B
    v9aXR              
   @   s  d Z dZddlZddlZddlmZmZ ddlT ddlm	Z	 dd	 e
e	D d
ddg Z[	ye W n ek
rz   eZY nX dd Zdd Zdd Zdd ZdZdZeede  ee ZdZdZdZdZeddZeeeeeZdZedd ee Zd!e ZeeeZed"ed# Z ee eeZ!d$Z"d%Z#d&Z$d'Z%d(Z&ee&d) e&d* Z'ee&d+ e&d, Z(ed-d.d/d0d1d2d3d4d5	Z)d6Z*ed7d8Z+ee)e*e+Z,ee!e,e(eZ-ee- Z.ee&d9 ed:d e&d; ed<d Z/edee'Z0eee0e!e,e/e Z1e2ej3e.e1e$e%f\Z4Z5Z6Z7ed=d>d?d@ed=d>dAdBB dCdDdEdFdGdHhB Z8e3e"e3e#e6e7dIdJdK e8D dLdK e8D dMdK e8D Z9d)d*hdNdO e8D B dPdO e8D B Z:d:d<hdQdO e8D B dRdO e8D B Z;dSZ<G dTdU dUe=Z>G dVdW dWe=Z?dXdY Z@e@fdZd
ZAd[d\ ZBG d]d^ d^ZCe3d_ejDZEe3d`ejDZFdadb ZGdcdd ZHded ZIdfd ZJeKdgkrddlLZLeMeLjNdkr~eAeOeLjNd jP neAeLjQjP dS )hzKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro    N)BOM_UTF8lookup)*   )tokenc             C   s   g | ]}|d  dkr|qS )r   _ ).0xr   r   /usr/lib/python3.7/tokenize.py
<listcomp>%   s    r   tokenizegenerate_tokens
untokenizec              G   s   dd |  d S )N(|))join)choicesr   r   r   group0       r   c              G   s   t |  d S )Nr   )r   )r   r   r   r   any1   r   r   c              G   s   t |  d S )N?)r   )r   r   r   r   maybe2   r   r   c                 s   t  fdd D S )Nc             3   s4   | ],} d  D ]}|  |  kr|| V  qqdS )) N)casefold)r	   r
   y)lr   r   	<genexpr>5   s    z _combinations.<locals>.<genexpr>)set)r   r   )r   r   _combinations3   s    r    z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"rRfFbBuUurZuRZUrZUR)r"   r#   z'''z"""c             C   s   i | ]}t | d qS )z''')single3prog)r	   prefixr   r   r   
<dictcomp>y   s    r/   c             C   s   i | ]}t | d qS )z""")double3prog)r	   r.   r   r   r   r/   z   s    c             C   s   i | ]
}d |qS )Nr   )r	   r.   r   r   r   r/   {   s    c             C   s   h | ]}| d qS )z'''r   )r	   r.   r   r   r   	<setcomp>   s    r1   c             C   s   h | ]}| d qS )z"""r   )r	   r.   r   r   r   r1      s    c             C   s   h | ]}| d qS )r"   r   )r	   r.   r   r   r   r1      s    c             C   s   h | ]}| d qS )r#   r   )r	   r.   r   r   r   r1      s       c               @   s   e Zd ZdS )
TokenErrorN)__name__
__module____qualname__r   r   r   r   r3      s    r3   c               @   s   e Zd ZdS )StopTokenizingN)r4   r5   r6   r   r   r   r   r7      s    r7   c       	   	   C   s4   |\}}|\}}t d||||t|  t|f  d S )Nz%d,%d-%d,%d:	%s	%s)printtok_namerepr)	typer   Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr   r   r   
printtoken   s    r=   c             C   s(   yt | | W n tk
r"   Y nX d S )N)tokenize_loopr7   )readline
tokeneaterr   r   r   r      s    c             C   s   xt | D ]}||  q
W d S )N)r   )r?   r@   Z
token_infor   r   r   r>      s    r>   c               @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc             C   s   g | _ d| _d| _d S )Nr   r   )tokensprev_rowprev_col)selfr   r   r   __init__   s    zUntokenizer.__init__c             C   s*   |\}}|| j  }|r&| jd|  d S )N )rD   rB   append)rE   startrowcol
col_offsetr   r   r   add_whitespace   s    
zUntokenizer.add_whitespacec             C   s   xv|D ]n}t |dkr$| || P |\}}}}}| | | j| |\| _| _|ttfkr|  jd7  _d| _qW d	| jS )N   r   r   r   )
lencompatrM   rB   rH   rC   rD   NEWLINENLr   )rE   iterablettok_typer   rI   endr<   r   r   r   r      s    


zUntokenizer.untokenizec       	      C   s   d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr<d}x|D ]}|d d \}}|ttttfkrn|d7 }|tkr|| qBn>|t	kr|
  qBn*|ttfkrd}n|r|r||d  d}|| qBW d S )NFrG   TrN   )rB   rH   NAMENUMBERrQ   rR   ASYNCAWAITINDENTDEDENTpop)	rE   r   rS   	startlineindentstoks_appendtoknumtokvaltokr   r   r   rP      s0    

zUntokenizer.compatN)r4   r5   r6   rF   rM   r   rP   r   r   r   r   rA      s   rA   z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$)c             C   sH   | d d   dd}|dks*|dr.dS |dks@|drDdS | S )	N   r   -zutf-8zutf-8-)zlatin-1z
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-z
iso-8859-1)lowerreplace
startswith)orig_encencr   r   r   _get_normal_name   s    
rl   c                s   d d }d}fdd} fdd}| }| trHd |dd  }d	}|sT|g fS ||}|rj||gfS t|s~||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c                  s"   y  S  t k
r   t S X d S )N)StopIterationbytesr   )r?   r   r   read_or_stop  s    z%detect_encoding.<locals>.read_or_stopc                s   y|  d}W n tk
r"   d S X t|}|s6d S t|d}yt|}W n  tk
rp   td| Y nX  r|j	dkrtd|d7 }|S )Nasciir   zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError	cookie_rematchrl   r   r   LookupErrorSyntaxErrorname)r<   line_stringrt   encodingcodec)	bom_foundr   r   find_cookie  s"    

z$detect_encoding.<locals>.find_cookieT   z	utf-8-sig)ri   r   blank_rert   )r?   ry   defaultro   r|   firstsecondr   )r{   r?   r   detect_encoding   s0    




r   c             C   s   t  }|| S )N)rA   r   )rS   utr   r   r   r   :  s    c             c   s  d } }}d\}}d }dg}d }d}	d}
d}xy
|  }W n t k
rT   d}Y nX |d }dt| }}|r6|std|||}|r|d }}t||d |  |||f|| fV  d\}}d }nd|r |dd  dkr |d	d  d
kr t|| ||t|f|fV  d}d }q2n|| }|| }q2nD|dkrb|sb|sNP d}xf||k r|| dkrv|d }n6|| dkr|t d t }n|| dkrd}nP |d }qTW ||krP |r|V  d }|| dkr|| dkrV||d  d}|t| }t	|||f||t| f|fV  t
||d  ||f|t|f|fV  q2t
t	f|| dk ||d  ||f|t|f|fV  q2||d kr|| t|d | |df||f|fV  xt||d k r8||krtdd|||f|d d }|	r|
|d krd}	d}d}
td||f||f|fV  qW |	rz|rz|
|d krzd}	d}d}
n|svtd|dfd}x||k r&t||}|r|d\}}||f||f|  }}}||| ||  }}|tjks|dkr
|dkr
t||||fV  q"|dkrVt}|dkr(t
}n
|	r2d}|rB|V  d }|||||fV  q"|dkr|rp|V  d }t	||||fV  q"|tkrt| }|||}|r|d}||| }|r|V  d }t||||f|fV  n||f}||d  }|}P q"|tks0|d d tks0|d d tkr|d dkr||f}t| pht|d  pht|d  }||d  d }}|}P n |r|V  d }t||||fV  q"| r~|dkr|	r|dkrtnt||||fV  q~t||||f}|dkr
|s
|}q~|dkrf|rf|d tkrf|d dkrfd}	|d }
t|d |d |d |d fV  d }|rv|V  d }|V  nz|dkr|r|V  d }t
||||f|fV  d}nF|dkr|d }n|d kr|d }|r|V  d }t||||fV  n(t|| ||f||d f|fV  |d }q~W q2W |r<|V  d }x.|dd  D ]}td|df|dfdfV  qJW td|df|dfdfV  d S )!Nr   )r   r   Fr   r   zEOF in multi-line stringz\
z\
rG   	z#
#z
rW   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.TrN   r}   
)asyncawaitr   def   \z([{z)]})rm   rO   r3   rt   rV   STRING
ERRORTOKENtabsizerstripCOMMENTrR   rH   r\   IndentationErrorr]   
pseudoprogspanstringdigitsrY   rQ   triple_quotedendprogssingle_quotedisidentifierrZ   r[   rX   OP	ENDMARKER)r?   lnumparenlev	continuedcontstrneedcontcontliner`   ZstashedZ	async_defZasync_def_indentZasync_def_nlr<   posmaxstrstartendprogendmatchrV   columncomment_tokenZnl_pospseudomatchrI   sposeposr   initialnewlinerd   indentr   r   r   r   O  sn   



*  
  
 
 











 

 __main__)R
__author____credits__r   recodecsr   r   Zlib2to3.pgen2.tokenr   r   dir__all__rn   	NameErrorstrr   r   r   r    
WhitespaceCommentIgnoreName	Binnumber	Hexnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberSingleDoubleSingle3Double3Z
_litprefixTripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStrPseudoExtrasPseudoTokenmapcompileZ	tokenprogr   r-   r0   Z_strprefixesr   r   r   r   	Exceptionr3   r7   r=   r   r>   rA   ASCIIrs   r~   rl   r   r   r   r4   sysrO   argvopenr?   stdinr   r   r   r   <module>   s   






8I b
 