관리-도구
편집 파일: tokenize.cpython-36.opt-2.pyc
3 \8s � - @ s� d Z dZddlmZ ddlmZmZ ddlZddl m Z ddl m Z ddl Z ddlZddlZddlT ejd ej�Zejd ej�ZddlZejd d d ddddg Z[eZd ee< ed Zdee< ed Zdee< ed7 Zee e e e e e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;e<e=e>e?e@eAeBeCeDeEeFd�,ZGG dd� dejHdd��ZIdd� ZJdd� ZKd d � ZLd ZMd ZNeMeKd eM � eLeN� ZOd!ZPd"ZQd#ZRd$ZSd%ZTeJeQeReSeT�ZUd&ZVeJd'd(�eLeV� ZWd)eV ZXeJeWeX�ZYeJd*eYd+ �ZZeJeZeYeU�Z[d,d-� Z\d.d/� Z]eJe\� � Z^d0Z_d1Z`d2Zad3ZbeJe^d4 e^d5 �ZceJe^d6 e^d7 �ZdeJd8d9d:d;d<d=d>d?�Zed@ZfeJdAdBdC�ZgeJeeefeg�ZheJe[ehedeP�ZieOei ZjeJe^dD eJdEd � e^dF eJdGd � �ZkeJdHeNec�ZleMeJele[ehekeP� Zmi Znx@e\� D ]6Zoe_eneodE < e`eneodG < eaeneod4 < ebeneod5 < �q�W ep� Zqep� Zrx\e\� D ]RZsx$esdG esdE fD ]Zteqjuet� �q4W x$esd5 esd4 fD ]Zterjuet� �qZW �q W dIZvG dJdK� dKew�ZxG dLdM� dMew�ZyG dNdO� dO�ZzdPd� Z{dQdR� Z|dSd � Z}dTdU� ZdVd � Z~dWdX� ZdYdZ� Z�d[d\� Z�e�d]k�r�e�� dS )^zKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)� TextIOWrapper)�chain)�*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)�COMMENT�tokenize�detect_encoding�NL� untokenize�ENCODING� TokenInfo� � � ),�(�)�[�]�:�,�;�+�-r �/�|�&�<�>�=�.�%�{�}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�@z@=c @ s e Zd Zdd� Zedd� �ZdS )r c C s$ d| j t| j f }d| j|d� S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r( �tok_name�_replace)�self�annotated_type� r- � /usr/lib64/python3.6/tokenize.py�__repr__d s zTokenInfo.__repr__c C s( | j tkr | jtkr t| j S | j S d S )N)r( �OP�string�EXACT_TOKEN_TYPES)r+ r- r- r. � exact_typei s zTokenInfo.exact_typeN)�__name__� __module__� __qualname__r/ �propertyr3 r- r- r- r. r c s ztype string start end linec G s ddj | � d S )Nr r r )�join)�choicesr- r- r. �groupp s r: c G s t | � d S )Nr )r: )r9 r- r- r. �anyq s r; c G s t | � d S )N�?)r: )r9 r- r- r. �mayber s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sn ddddddg} t dg�}xN| D ]F}x@tj|�D ]2}x,tjdd � |D �� D ]}|jdj|�� qJW q0W q W |S ) N�b�r�u�f�br�fr� c S s g | ]}||j � f�qS r- )�upper)�.0�cr- r- r. � <listcomp>� s z(_all_string_prefixes.<locals>.<listcomp>)�set� _itertools� permutations�product�addr8 )�_valid_string_prefixes�result�prefix�tr@ r- r- r. �_all_string_prefixes� s rR c C s t j| t j�S )N)�re�compile�UNICODE)�exprr- r- r. �_compile� s rW z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r% z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z \\\r?\n|\Z� c @ s e Zd ZdS )� TokenErrorN)r4 r5 r6 r- r- r- r. r[ � s r[ c @ s e Zd ZdS )�StopTokenizingN)r4 r5 r6 r- r- r- r. r\ � s r\ c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � Untokenizerc C s g | _ d| _d| _d | _d S )Nr r )�tokens�prev_row�prev_col�encoding)r+ r- r- r. �__init__� s zUntokenizer.__init__c C s� |\}}|| j k s&|| j kr>|| jk r>tdj||| j | j���|| j }|rb| jjd| � d| _|| j }|r�| jjd| � d S )Nz+start ({},{}) precedes previous end ({},{})z\ r � )r_ r` � ValueError�formatr^ �append)r+ �start�row�col� row_offset� col_offsetr- r- r. �add_whitespace� s zUntokenizer.add_whitespacec C s4 t |�}g }d}�x|D �] }t|�dkr8| j||� P |\}}}} } |tkrV|| _q|tkr`P |tkrv|j|� qnl|tkr�|j � | \| _ | _ qnL|t t fkr�d}n:|r�|r�|d } |d t| �kr�| jj| � t| �| _ d}| j|� | jj|� | \| _ | _ |t t fkr| j d7 _ d| _ qW dj| j�S )NFr Tr r rD ���)�iter�len�compatr ra � ENDMARKER�INDENTrf �DEDENT�popr_ r` �NEWLINEr r^ rl r8 ) r+ �iterable�it�indents� startlinerQ �tok_type�tokenrg �end�line�indentr- r- r. r � sF zUntokenizer.untokenizec C s� g }| j j}|d ttfk}d}x�t|g|�D ]�}|d d� \}} |tkrR| | _q.|ttt t fkrj| d7 } |t kr�|r~d| } d}nd}|t kr�|j| � q.n>|t kr�|j� q.n*|ttfkr�d}n|r�|r�||d � d}|| � q.W d S )Nr Fr rc Tr rm )r^ rf ru r r r ra �NAME�NUMBER�ASYNC�AWAIT�STRINGrr rs rt ) r+ r{ rv rx � toks_appendry � prevstring�tok�toknum�tokvalr- r- r. rp s8 zUntokenizer.compatN)r4 r5 r6 rb rl r rp r- r- r- r. r] � s %r] c C s* t � }|j| �}|jd k r&|j|j�}|S )N)r] r ra �encode)rv �ut�outr- r- r. r = s c C sH | d d� j � jdd�}|dks*|jd�r.dS |d ks@|jd �rDdS | S )N� �_r zutf-8zutf-8-�latin-1� iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace� startswith)�orig_enc�encr- r- r. �_get_normal_nameX s r� c s� y �j j�W n tk r$ d �Y nX d� d }d}�fdd�}� �fdd�}|� }|jt�rpd� |dd � }d }|s||g fS ||�}|r�||gfS tj|�s�||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS ) NFzutf-8c s y� � S t k r dS X d S )N� )� StopIterationr- )�readliner- r. � read_or_stop{ s z%detect_encoding.<locals>.read_or_stopc s� y| j d�}W n4 tk rB d}�d k r6dj|��}t|��Y nX tj|�}|sVd S t|jd��}y t|�}W n: t k r� �d kr�d| }n dj�|�}t|��Y nX � r�|dkr؈d kr�d}n dj��}t|��|d 7 }|S ) Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) �decode�UnicodeDecodeErrorre � SyntaxError� cookie_re�matchr� r: r � LookupError)r} � line_string�msgr� ra �codec)� bom_found�filenamer- r. � find_cookie� s6 z$detect_encoding.<locals>.find_cookieTr z utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� ra �defaultr� r� �first�secondr- )r� r� r� r. r c s8 & c C sV t | d�}y0t|j�\}}|jd� t||dd�}d|_|S |j� � Y nX d S )N�rbr T)�line_bufferingr? )� _builtin_openr r� �seekr �mode�close)r� �bufferra �lines�textr- r- r. r � s r c C sB ddl m}m} t| �\}}t| d�}|d�}t||||�j|�S )Nr )r �repeatr� )� itertoolsr r� r rn � _tokenize�__next__)r� r r� ra �consumed�rl_gen�emptyr- r- r. r � s c ! c s d } }}d}d!\}}d }dg} d } d} d} d} |d k rX|dkrFd}t t|d"d#d�V d}d}�x y|}| � }W n tk r� d}Y nX |d k r�|j|�}|d7 }dt|� }}|�r�|s�td |��|j|�}|�r|jd� }}t t||d |� |||f|| �V d$\}}d }nf|�rn|d%d � d k�rn|d&d � d k�rnt t || ||t|�f|�V d}d }qdn|| }|| }qd�n@|dk�r�| �r�|�s�P d}xf||k �r|| dk�r�|d7 }n6|| dk�r�|t d t }n|| dk�r�d}nP |d7 }�q�W ||k�rP || dk�r�|| dk�r�||d � j d�}|t|� }t t |||f||t|� f|�V t t ||d � ||f|t|�f|�V qdt t t f|| dk ||d � ||f|t|�f|�V qd|| d' k�r | j|� t t|d |� |df||f|�V xv|| d( k �r�|| k�r8tdd|||f��| d d)� } | �rd| | d* k�rdd} d} d} t td||f||f|�V �qW | �r�| �r�| | d+ k�r�d} d} d} n|�s�td|df��d}�x�||k �rhtt�j||�}|�r:|jd�\}}||f||f| }} }||k�r �q�|||� || } } | |k�sZ| dk�rp| dk�rp| dk�rpt t| || |�V �qd| dk�r�| �r�| V d } |dk�r�t t | || |�V n t t| || |�V | �r8d} �qd| dk�r�| �r�| V d } t t | || |�V �qd| tk�rltt| �}|j||�}|�rN|jd�}|||� } t t| |||f|�V n||f}||d � }|}P �qd| tk�s�| d d � tk�s�| d d � tk�r| d, dk�r�||f}ttj| ��p�tj| d ��p�tj| d ��}||d � d }}|}P nt t| || |�V �qd| j� �r�| d-k�rL| �rLt | dk�r8t nt | || |�V �q�t t | || |�} | dk�rv| �rv| } �q�| d k�r�| �r�| j t k�r�| j dk�r�d} | d. } t t | j | j!| j| j"�V d } | �r�| V d } | V nX| d k�r�d}nH| d k�r|d7 }n| d k�r|d8 }| �r&| V d } t t#| || |�V n*t t || ||f||d f|�V |d7 }�q�W qdW | �r~| V d } |�r�|d/ dk�r�t td|d t|�f|d t|�d fd�V x0| dd � D ] } t td|df|dfd�V �q�W t t$d|df|dfd�V d S )0Nr � 0123456789rD Fz utf-8-sigzutf-8r� r zEOF in multi-line stringr z\ r z\ rc � � z# �#z z3unindent does not match any outer indentation levelz <tokenize>zEOF in multi-line statementr! z...T� �async�await�def�\z([{z)]})rD r )r r )r r )rD r ������rm rm rm rm rm rm )r� r� rm rm )%r r r� r� ro r[ r� r| r� � ERRORTOKEN�tabsize�rstripr r rf rr �IndentationErrorrs rW � PseudoToken�spanr� ru � triple_quoted�endpats� single_quoted�get� isidentifierr� r� r r( r1 rg r} r0 rq )!r� ra �lnum�parenlev� continued�numchars�contstr�needcont�contlinerx �stashed� async_def�async_def_indent� async_def_nl� last_liner} �pos�max�strstart�endprog�endmatchr| �column� comment_token�nl_pos� pseudomatchrg �spos�eposr{ �initialr� r~ r- r- r. r� � sr * " . r� c C s t | d �S )N)r� )r� r- r- r. �generate_tokens� s r� c s( dd l } dd� � d� fdd� }| jdd�}|jdd d d d � |jd ddddd� |j� }y�|jr�|j}t|d��}tt|j��}W d Q R X nd}t t j jd �}xF|D ]>}|j }|j r�|j}d|j|j } td| t| |jf � q�W W �n8 tk �r: } z2| jd dd� \} } || jd || | f� W Y d d } ~ X n� tk �r� } z*| jd \} } || jd || | f� W Y d d } ~ X n� tk �r� } z|| |� W Y d d } ~ X nx tk �r� } z|| � W Y d d } ~ X nN tk �r� td� Y n2 tk �r" } z� d| � � W Y d d } ~ X nX d S ) Nr c S s t | tjd� d S )N)�file)�print�sys�stderr)�messager- r- r. �perror� s zmain.<locals>.perrorc sR |r"|f| | f }� d| � n"|r8� d|| f � n � d| � t jd� d S )Nz%s:%d:%d: error: %sz %s: error: %sz error: %sr )r� �exit)r� r� �location�args)r� r- r. �error� s zmain.<locals>.errorzpython -m tokenize)�progr� r< z filename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact� store_truez(display token names using the exact type)r� �actionr� r� z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted zunexpected error: %s)NN)�argparse�ArgumentParser� add_argument� parse_argsr� r� �listr r� r� r� �stdinr( r� r3 rg r| r� r) r1 r� r� r[ r� �OSError�KeyboardInterrupt� Exception) r r� �parserr� r� rA r^ r{ � token_type� token_range�errr} r� r- )r� r. �main� sN && r �__main__)�� __author__� __credits__�builtinsr r� �codecsr r � collections�ior r� r rJ rS r� r{ rT �ASCIIr� r� �__all__�N_TOKENSr r) r r �LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE� CIRCUMFLEX� LEFTSHIFT� RIGHTSHIFT� DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL� SLASHEQUAL� PERCENTEQUAL� AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALr2 � namedtupler r: r; r= � Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent� Pointfloat�Expfloat� Floatnumber� Imagnumber�NumberrR rW � StringPrefix�Single�Double�Single3�Double3�Triple�String�Operator�Bracket�Special�Funny� PlainToken�Token�ContStr� PseudoExtrasr� r� �_prefixrI r� r� rQ r@ rM r� r r[ r\ r] r r� r r r� r� r r4 r- r- r- r. �<module> s _ ] x<