B
    u9ai              /   @   s  d Z dZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ ddlZddlZddlZddlT ed	ejZed
ejZddlZejddddg Z[eeeeeeeeeee e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;e<e=e>e?e@eAeBeCd.ZDG dd deEddZFdd ZGdd ZHdd ZIdZJdZKeJeHdeJ  eIeK ZLdZMdZNdZOdZPdZQeGeNeOePeQZRd ZSeGd!d"eIeS ZTd#eS ZUeGeTeUZVeGd$eVd% ZWeGeWeVeRZXd&d' ZYd(d) ZZeGeY  Z[d*Z\d+Z]d,Z^d-Z_eGe[d. e[d/ Z`eGe[d0 e[d1 ZaeGd2d3d4d5d6d7d8d9Zbd:ZceGd;d<d=ZdeGebecedZeeGeXeeeaeMZfeLef ZgeGe[d> eGd?d e[d@ eGdAd ZheGdBeKe`ZieJeGeieXeeeheM Zji Zkx@eY D ]6Zle\ekeld? < e]ekeldA < e^ekeld. < e_ekeld/ < qW em Znem Zox\eY D ]RZpx$epdA epd? fD ]Zqenreq qW x$epd/ epd. fD ]Zqeoreq q$W qW dCZsG dDdE dEetZuG dFdG dGetZvG dHdI dIZwdJd ZxdKdL ZydMd ZzdNdO ZdPd Z{dQdR Z|dSdT Z}dUdV Z~edWkre~  dS )XzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord    )open)lookupBOM_UTF8N)TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$)tokenizedetect_encoding
untokenize	TokenInfo).()[]:,;+-r   /|&<>=.%{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z...z->@z@=c               @   s    e Zd Zdd Zedd ZdS )r   c             C   s$   d| j t| j  f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r"   tok_name_replace)selfannotated_type r'   /usr/lib/python3.7/tokenize.py__repr___   s    zTokenInfo.__repr__c             C   s(   | j tkr| jtkrt| j S | j S d S )N)r"   OPstringEXACT_TOKEN_TYPES)r%   r'   r'   r(   
exact_typed   s    
zTokenInfo.exact_typeN)__name__
__module____qualname__r)   propertyr-   r'   r'   r'   r(   r   ^   s   ztype string start end linec              G   s   dd |  d S )Nr   r   r   )join)choicesr'   r'   r(   groupk       r4   c              G   s   t |  d S )Nr   )r4   )r3   r'   r'   r(   anyl   r5   r6   c              G   s   t |  d S )N?)r4   )r3   r'   r'   r(   maybem   r5   r8   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c              C   sj   ddddddg} dh}xN| D ]F}x@t |D ]2}x,t jdd	 |D  D ]}|d| qFW q,W qW |S )
Nbrufbrfr c             S   s   g | ]}||  fqS r'   )upper).0cr'   r'   r(   
<listcomp>   s    z(_all_string_prefixes.<locals>.<listcomp>)
_itertoolspermutationsproductaddr2   )_valid_string_prefixesresultprefixtr;   r'   r'   r(   _all_string_prefixes   s    
rL   c             C   s   t | t jS )N)recompileUNICODE)exprr'   r'   r(   _compile   s    rQ   z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r   z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z   c               @   s   e Zd ZdS )
TokenErrorN)r.   r/   r0   r'   r'   r'   r(   rU      s    rU   c               @   s   e Zd ZdS )StopTokenizingN)r.   r/   r0   r'   r'   r'   r(   rV      s    rV   c               @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc             C   s   g | _ d| _d| _d | _d S )N   r   )tokensprev_rowprev_colencoding)r%   r'   r'   r(   __init__   s    zUntokenizer.__init__c             C   s   |\}}|| j k s&|| j kr>|| jk r>td||| j | j|| j  }|rb| jd|  d| _|| j }|r| jd|  d S )Nz+start ({},{}) precedes previous end ({},{})z\
r    )rZ   r[   
ValueErrorformatrY   append)r%   startrowcol
row_offset
col_offsetr'   r'   r(   add_whitespace   s    

zUntokenizer.add_whitespacec             C   s4  t |}g }d}x|D ]
}t|dkr8| || P |\}}}}	}
|tkrV|| _q|tkr`P |tkrv|| qnl|tkr|	  |	\| _
| _qnL|ttfkrd}n:|r|r|d }|d t|kr| j| t|| _d}| | | j| |	\| _
| _|ttfkr|  j
d7  _
d| _qW d| jS )NF   TrX   r   r?   )iterlencompatENCODINGr\   	ENDMARKERINDENTra   DEDENTpoprZ   r[   NEWLINENLrY   rg   r2   )r%   iterableitindents	startlinerK   tok_typetokenrb   endlineindentr'   r'   r(   r
      sF    



zUntokenizer.untokenizec       
      C   s   g }| j j}|d ttfk}d}xt|g|D ]}|d d \}}	|tkrR|	| _q.|ttfkrf|	d7 }	|t	kr|rzd|	 }	d}nd}|t
kr||	 q.n>|tkr|  q.n*|ttfkrd}n|r|r||d  d}||	 q.W d S )Nr   Frh   r^   Tri   )rY   ra   rr   rs   r   rm   r\   NAMENUMBERSTRINGro   rp   rq   )
r%   ry   rt   rv   toks_appendrw   
prevstringtoktoknumtokvalr'   r'   r(   rl     s8    
zUntokenizer.compatN)r.   r/   r0   r]   rg   r
   rl   r'   r'   r'   r(   rW      s   %rW   c             C   s*   t  }|| }|jd k	r&||j}|S )N)rW   r
   r\   encode)rt   utoutr'   r'   r(   r
   8  s
    

c             C   sH   | d d   dd}|dks*|dr.dS |dks@|drDdS | S )	N   _r   zutf-8zutf-8-)zlatin-1z
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-z
iso-8859-1)lowerreplace
startswith)orig_encencr'   r'   r(   _get_normal_nameS  s    
r   c                s   yj jW n tk
r$   d Y nX d d }d}fdd} fdd}| }|trpd |dd  }d	}|s||g fS ||}|r||gfS t|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c                  s    y  S  t k
r   dS X d S )Nr5   )StopIterationr'   )readliner'   r(   read_or_stopv  s    z%detect_encoding.<locals>.read_or_stopc                s   y|  d}W n4 tk
rB   d}d k	r6d|}t|Y nX t|}|sVd S t|d}yt|}W n: t	k
r   d krd| }nd|}t|Y nX  r|dkr؈d krd}n
d}t||d	7 }|S )
Nzutf-8z'invalid or missing encoding declarationz{} for {!r}rX   zunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorr`   SyntaxError	cookie_rematchr   r4   r   LookupError)r{   line_stringmsgr   r\   codec)	bom_foundfilenamer'   r(   find_cookie|  s6    


z$detect_encoding.<locals>.find_cookieT   z	utf-8-sig)__self__nameAttributeErrorr   r   blank_rer   )r   r\   defaultr   r   firstsecondr'   )r   r   r   r(   r	   ^  s8    
&




c             C   sV   t | d}y0t|j\}}|d t||dd}d|_|S    |   Y nX d S )Nrbr   T)line_bufferingr:   )_builtin_openr	   r   seekr   modeclose)r   bufferr\   linestextr'   r'   r(   r     s    

r   c             C   sB   ddl m}m} t| \}}t| d}|d}t||||j|S )Nr   )r   repeatr5   )	itertoolsr   r   r	   rj   	_tokenize__next__)r   r   r   r\   consumedrl_genemptyr'   r'   r(   r     s
    
c             c   s  d } }}d}d\}}d }dg}	|d k	rH|dkr6d}t t|dddV  d}
d}xy|}
|  }W n tk
rz   d}Y nX |d k	r||}|d	7 }dt| }}|rt|std
|||}|r
|d }}t t||d |  |||f|| V  d\}}d }nf|r^|dd  dkr^|dd  dkr^t t	|| ||t|f|V  d}d }qTn|| }|| }qTn|dkr|s|sP d}xf||k r|| dkr|d	7 }n6|| dkr|t
 d	 t
 }n|| dkrd}nP |d	7 }qW ||krP || dkr|| dkr`||d  d}t t|||f||t| f|V  |t|7 }t t||d  ||f|t|f|V  qT||	d kr|	| t t|d | |df||f|V  xp||	d k r||	krtdd|||f|	d d }	t td||f||f|V  qW n|s0td|dfd}x||k rtt||}|r|d	\}}||f||f|  }}}||krq8||| ||  }}||ks|dkr|dkr|dkrt t||||V  q|dkr|dkrt t||||V  nt t||||V  q|dkr>t t||||V  q|tkrtt| }|||}|r|d}||| }t t||||f|V  n||f}||d  }|}P q|tks|d d tks|d d tkrR|d dkr>||f}tt|p t|d	 p t|d }||d  d	 }}|}P nt t||||V  nf| rpt t||||V  nH|dkrd	}n8|dkr|d	7 }n|d kr|d	8 }t t||||V  n*t t	|| ||f||d	 f|V  |d	7 }q8W qTW |
r0|
d dkr0t td|d	 t|
f|d	 t|
d	 fdV  x0|	d	d  D ] }t td|df|dfdV  q>W t td|df|dfdV  d S )!Nr   
0123456789)r?   r   z	utf-8-sigzutf-8)r   r   r?   r5   rX   zEOF in multi-line stringz\
z\
r^   	z#
#z
ri   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statementr   z...rh   r   
\z([{z)]})r   rm   r   r   rk   rU   r   rz   r   
ERRORTOKENtabsizerstripCOMMENTrs   ra   ro   IndentationErrorrp   rQ   PseudoTokenspanr~   rr   triple_quotedendpatssingle_quotedgetisidentifierr}   r*   rn   )r   r\   lnumparenlev	continuednumcharscontstrneedcontcontlinerv   	last_liner{   posmaxstrstartendprogendmatchrz   columncomment_tokenpseudomatchrb   sposeposry   initialr|   r'   r'   r(   r     s   




*
 


"
"













. r   c             C   s
   t | d S )N)r   )r   r'   r'   r(   generate_tokens  s    r   c           
      s(  dd l } dd  d fdd	}| jdd}|jdd	d
dd |jdddddd | }y|jr|j}t|d}tt|j}W d Q R X nd}t	t
jjd }xF|D ]>}|j}|jr|j}d|j|j  }	td|	t| |jf  qW W n8 tk
r: }
 z0|
jd dd \}}||
jd |||f W d d }
~
X Y n tk
r }
 z(|
jd \}}||
jd |||f W d d }
~
X Y n tk
r }
 z||
| W d d }
~
X Y nx tk
r }
 z||
 W d d }
~
X Y nN tk
r   td Y n2 tk
r" }
 z d|
   W d d }
~
X Y nX d S )Nr   c             S   s   t | tjd d S )N)file)printsysstderr)messager'   r'   r(   perror  s    zmain.<locals>.perrorc                sR   |r"|f| | f } d|  n"|r8 d|| f  n d|   t d d S )Nz%s:%d:%d: error: %sz%s: error: %sz	error: %srX   )r   exit)r   r   locationargs)r   r'   r(   error  s    zmain.<locals>.errorzpython -m tokenize)progr   r7   zfilename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r   actionr   r   z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrX   r   zinterrupted
zunexpected error: %s)NN)argparseArgumentParseradd_argument
parse_argsr   r   listr   r   r   r   stdinr"   r   r-   rb   rz   r   r#   r+   r   r   rU   r   OSErrorKeyboardInterrupt	Exception)r   r   parserr   r   r<   rY   ry   
token_typetoken_rangeerrr{   r   r'   )r   r(   main  sN    
&&r   __main__)
__author____credits__builtinsr   r   codecsr   r   collectionsior   r   r   rD   rM   r   ry   rN   ASCIIr   r   __all__LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL	LESSEQUALGREATEREQUALTILDE
CIRCUMFLEX	LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR	PLUSEQUALMINEQUAL	STAREQUAL
SLASHEQUALPERCENTEQUAL
AMPEREQUAL	VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUALDOUBLESLASHDOUBLESLASHEQUALELLIPSISRARROWATATEQUALr,   
namedtupler   r4   r6   r8   
WhitespaceCommentIgnoreName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberrL   rQ   StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStrPseudoExtrasr   r   _prefixsetr   r   rK   r;   rG   r   r   rU   rV   rW   r
   r   r	   r   r   r   r   r.   r'   r'   r'   r(   <module>   s  





_] :<
