a
    zeR                  
   @   s  d Z dZddlZddlZddlmZmZ ddlT ddlm	Z	 dd	 e
e	D g d
 Z[	ze W n eyv   eZY n0 dd Zdd Zdd Zdd ZdZdZeede  ee ZdZdZdZdZeddZeeeeeZdZeddee Zde ZeeeZed ed! Z ee eeZ!d"Z"d#Z#d$Z$d%Z%d&Z&ee&d' e&d( Z'ee&d) e&d* Z(ed+d,d-d.d/d0d1d2d3	Z)d4Z*ed5d6d7Z+ee)e*e+Z,ee!e,e(eZ-ee- Z.ee&d8 ed9d e&d: ed;d Z/edee'Z0eee0e!e,e/e Z1e2ej3e.e1e$e%f\Z4Z5Z6Z7ed<d=d>d?ed<d=d@dAB h dBB Z8e3e"e3e#e6e7dCdDdE e8D dFdE e8D dGdE e8D Z9d'd(hdHdI e8D B dJdI e8D B Z:d9d;hdKdI e8D B dLdI e8D B Z;dMZ<G dNdO dOe=Z>G dPdQ dQe=Z?dRdS Z@e@fdTdUZAdVdW ZBG dXdY dYZCe3dZejDZEe3d[ejDZFd\d] ZGd^d_ ZHd`da ZIdbdc ZJeKddkrddlLZLeMeLjNdkrxeAeOeLjNd jP neAeLjQjP dS )ezKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro    N)BOM_UTF8lookup)*   )tokenc                 C   s   g | ]}|d  dkr|qS )r   _ ).0xr   r   ,/usr/lib/python3.9/lib2to3/pgen2/tokenize.py
<listcomp>%       r   )tokenizegenerate_tokens
untokenizec                  G   s   dd |  d S )N(|))joinchoicesr   r   r   group0   r   r   c                  G   s   t |  d S )Nr   r   r   r   r   r   any1   r   r   c                  G   s   t |  d S )N?r   r   r   r   r   maybe2   r   r   c                     s   t  fdd D S )Nc                 3   s4   | ],} d  D ]}|  |  kr|| V  qqdS )) N)casefold)r	   r
   ylr   r   	<genexpr>4   s   z _combinations.<locals>.<genexpr>)setr   r   r   r   _combinations3   s    r#   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?'''"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"rRfFbB>   urUuZuRZURZUr)r'   r(   r$   r%   c                 C   s   i | ]}| d t qS r$   )single3progr	   prefixr   r   r   
<dictcomp>y   r   r6   c                 C   s   i | ]}| d t qS r%   )double3progr4   r   r   r   r6   z   r   c                 C   s   i | ]
}|d qS Nr   r4   r   r   r   r6   {   r   c                 C   s   h | ]}| d qS r2   r   r4   r   r   r   	<setcomp>   r   r:   c                 C   s   h | ]}| d qS r7   r   r4   r   r   r   r:      r   c                 C   s   h | ]}| d qS )r'   r   r4   r   r   r   r:      r   c                 C   s   h | ]}| d qS )r(   r   r4   r   r   r   r:      r      c                   @   s   e Zd ZdS )
TokenErrorN__name__
__module____qualname__r   r   r   r   r<      r   r<   c                   @   s   e Zd ZdS )StopTokenizingNr=   r   r   r   r   rA      r   rA   c           	   	   C   s4   |\}}|\}}t d||||t|  t|f  d S )Nz%d,%d-%d,%d:	%s	%s)printtok_namerepr)	typer   Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr   r   r   
printtoken   s
    rG   c                 C   s&   zt | | W n ty    Y n0 d S r9   )tokenize_looprA   )readline
tokeneaterr   r   r   r      s    r   c                 C   s   t | D ]}||  qd S r9   )r   )rI   rJ   Z
token_infor   r   r   rH      s    rH   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc                 C   s   g | _ d| _d| _d S )Nr   r   )tokensprev_rowprev_col)selfr   r   r   __init__   s    zUntokenizer.__init__c                 C   s*   |\}}|| j  }|r&| jd|  d S )N )rN   rL   append)rO   startrowcol
col_offsetr   r   r   add_whitespace   s    
zUntokenizer.add_whitespacec                 C   s   |D ]p}t |dkr$| ||  qv|\}}}}}| | | j| |\| _| _|ttfv r|  jd7  _d| _qd	| jS )N   r   r   r   )
lencompatrW   rL   rR   rM   rN   NEWLINENLr   )rO   iterablettok_typer   rS   endrF   r   r   r   r      s    
zUntokenizer.untokenizec           	      C   s   d}g }| j j}|\}}|ttfv r,|d7 }|ttfv r<d}|D ]}|d d \}}|ttttfv rl|d7 }|tkr|| q@n>|t	kr|
  q@n*|ttfv rd}n|r|r||d  d}|| q@d S )NFrQ   TrX   )rL   rR   NAMENUMBERr[   r\   ASYNCAWAITINDENTDEDENTpop)	rO   r   r]   	startlineindentstoks_appendtoknumtokvaltokr   r   r   rZ      s0    
zUntokenizer.compatN)r>   r?   r@   rP   rW   r   rZ   r   r   r   r   rK      s   rK   z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$)c                 C   sH   | d d   dd}|dks*|dr.dS |dv s@|drDdS | S )	N   r   -utf-8zutf-8-)zlatin-1
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-rr   )lowerreplace
startswith)orig_encencr   r   r   _get_normal_name   s    rx   c                    s   d d }d}fdd} fdd}| }| trHd |dd  }d	}|sT|g fS ||}|rj||gfS t|s~||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFrq   c                      s&   z  W S  t y    t  Y S 0 d S r9   )StopIterationbytesr   )rI   r   r   read_or_stop  s    z%detect_encoding.<locals>.read_or_stopc                    s   z|  d}W n ty"   Y d S 0 t|}|s6d S t|d}zt|}W n tyn   td| Y n0  r|j	dkrtd|d7 }|S )Nasciir   zunknown encoding: rq   zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError	cookie_rematchrx   r   r   LookupErrorSyntaxErrorname)rF   line_stringr   encodingcodec)	bom_foundr   r   find_cookie  s"    

z$detect_encoding.<locals>.find_cookieT   z	utf-8-sig)ru   r   blank_rer   )rI   r   defaultr{   r   firstsecondr   )r   rI   r   detect_encoding   s0    




r   c                 C   s   t  }|| S r9   )rK   r   )r]   utr   r   r   r   :  s    r   c                 c   s|  d } }}d\}}d }dg}d }d}	d}
d}z
|  }W n t yN   d}Y n0 |d }dt| }}|r0|sztd|||}|r|d }}t||d |  |||f|| fV  d\}}d }nd|r|dd  dkr|d	d  d
krt|| ||t|f|fV  d}d }q.n|| }|| }q.nB|dkrZ|sZ|sJq$d}||k r|| dkrp|d }n8|| dkr|t d t }n|| dkrd}nq|d }qN||krq$|r|V  d }|| dv r|| dkrR||d  d}|t| }t	|||f||t| f|fV  t
||d  ||f|t|f|fV  q.t
t	f|| dk ||d  ||f|t|f|fV  q.||d kr|| t|d | |df||f|fV  ||d k r2||vrtdd|||f|d d }|	r|
|d krd}	d}d}
td||f||f|fV  q|	rr|rr|
|d krrd}	d}d}
n|sntd|dfd}||k r.t||}|r|d\}}||f||f|  }}}||| ||  }}|tjv s|dkr|dkrt||||fV  q|dv rHt}|dkrt
}n
|	r$d}|r4|V  d }|||||fV  q|dkrv|rb|V  d }t	||||fV  q|tv rt| }|||}|r|d}||| }|r|V  d }t||||f|fV  n||f}||d  }|}q.q|tv s"|d d tv s"|d d tv r|d dkrv||f}t| pZt|d  pZt|d  }||d  d }}|}q.n |r|V  d }t||||fV  q| rz|dv r|	r|dkrtnt||||fV  qrt||||f}|dkr|s|}qr|dv rb|rb|d tkrb|d dkrb|dkr>d}	|d }
t|d |d |d |d fV  d }|rr|V  d }|V  nz|dkr|r|V  d }t
||||f|fV  d}nF|d v r|d }n|d!v r|d }|r|V  d }t||||fV  n(t|| ||f||d f|fV  |d }qrq.|r4|V  d }|dd  D ]}td|df|dfdfV  q@td|df|dfdfV  d S )"Nr   )r   r   Fr   r   zEOF in multi-line stringz\
z\
rQ   	z#
#z
ra   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.TrX   r   
)asyncawaitr   )defforr      \z([{z)]})ry   rY   r<   r   r`   STRING
ERRORTOKENtabsizerstripCOMMENTr\   rR   rf   IndentationErrorrg   
pseudoprogspanstringdigitsrc   r[   triple_quotedendprogssingle_quotedisidentifierrd   re   rb   OP	ENDMARKER)rI   lnumparenlev	continuedcontstrneedcontcontlinerj   ZstashedZ	async_defZasync_def_indentZasync_def_nlrF   posmaxstrstartendprogendmatchr`   columncomment_tokenZnl_pospseudomatchrS   sposeposr   initialnewlinern   indentr   r   r   r   O  s   



*

 
 















r   __main__)R
__author____credits__r   recodecsr   r   Zlib2to3.pgen2.tokenr   r   dir__all__rz   	NameErrorstrr   r   r   r#   
WhitespaceCommentIgnoreName	Binnumber	Hexnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberSingleDoubleSingle3Double3Z
_litprefixTripleStringZOperatorZBracketSpecialFunny
PlainTokenTokenContStrPseudoExtrasPseudoTokenmapcompileZ	tokenprogr   r3   r8   Z_strprefixesr   r   r   r   	Exceptionr<   rA   rG   r   rH   rK   ASCIIr   r   rx   r   r   r   r>   sysrY   argvopenrI   stdinr   r   r   r   <module>   s   


8I c
&