a
    zee                     @   s>  d Z dZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlZddlZddlZddlT ddlmZ ed	ejZed
ejZddlZejg d Z[G dd deddZdd Zdd Zdd ZdZdZeede  ee ZdZdZdZ dZ!dZ"eee e!e"Z#dZ$eddee$ Z%d e$ Z&ee%e&Z'ed!e'd" Z(ee(e'e#Z)d#d$ Z*d%d& Z+ee*  Z,d'Z-d(Z.d)Z/d*Z0ee,d+ e,d, Z1ee,d- e,d. Z2ee3ej4e5ed/d0 Z6ed1e6Z7ee)e7e2eZ8ee8 Z9ee,d2 ed3d e,d4 ed5d Z:ed6ee1Z;eee;e)e7e:e Z<i Z=e* D ]6Z>e-e=e>d3 < e.e=e>d5 < e/e=e>d+ < e0e=e>d, < q"e? Z@e? ZAe* D ]JZBeBd5 eBd3 fD ]ZCe@DeC qeBd, eBd+ fD ]ZCeADeC qqld7ZEG d8d9 d9eFZGG d:d; d;eFZHG d<d= d=ZId>d? ZJd@dA ZKdBdC ZLdDdE ZdFdG ZMdHdI ZNdJdK ZOdLdM ZPeQdNkr:eP  dS )OzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord    )open)lookupBOM_UTF8N)TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s   ^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding
untokenize	TokenInfoc                   @   s    e Zd Zdd Zedd ZdS )r   c                 C   s$   d| j t| j  f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r   tok_name_replace)selfannotated_type r   /usr/lib/python3.9/tokenize.py__repr__.   s    
zTokenInfo.__repr__c                 C   s(   | j tkr| jtv rt| j S | j S d S N)r   OPstringr   r   r   r   r   
exact_type3   s    
zTokenInfo.exact_typeN)__name__
__module____qualname__r   propertyr   r   r   r   r   r   -   s   r   ztype string start end linec                  G   s   dd |  d S )N(|))joinchoicesr   r   r   group:       r$   c                  G   s   t |  d S )Nr   r$   r"   r   r   r   any;   r%   r'   c                  G   s   t |  d S )N?r&   r"   r   r   r   maybe<   r%   r)   z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c                  C   sV   g d} dh}| D ]>}t |D ].}t jdd |D  D ]}|d| q8q q|S )N)brufbrfr c                 S   s   g | ]}||  fqS r   )upper).0cr   r   r   
<listcomp>^   r%   z(_all_string_prefixes.<locals>.<listcomp>)
_itertoolspermutationsproductaddr!   )_valid_string_prefixesresultprefixtr,   r   r   r   _all_string_prefixesS   s    r=   c                 C   s   t | t jS r   )recompileUNICODE)exprr   r   r   _compileb   s    rB   z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z   c                   @   s   e Zd ZdS )
TokenErrorNr   r   r   r   r   r   r   rG      r%   rG   c                   @   s   e Zd ZdS )StopTokenizingNrH   r   r   r   r   rI      r%   rI   c                   @   s,   e Zd Zdd Zdd Zdd Zdd Zd	S )
Untokenizerc                 C   s   g | _ d| _d| _d | _d S )N   r   )tokensprev_rowprev_colencodingr   r   r   r   __init__   s    zUntokenizer.__init__c                 C   s   |\}}|| j k s&|| j kr>|| jk r>td||| j | j|| j  }|rb| jd|  d| _|| j }|r| jd|  d S )Nz+start ({},{}) precedes previous end ({},{})\
r    )rM   rN   
ValueErrorformatrL   append)r   startrowcol
row_offset
col_offsetr   r   r   add_whitespace   s    

zUntokenizer.add_whitespacec                 C   s6  t |}g }d}|D ]}t|dkr8| ||  q*|\}}}}	}
|tkrV|| _q|tkrd q*|tkrz|| qnl|tkr|	  |	\| _
| _qnL|ttfv rd}n:|r|r|d }|d t|kr| j| t|| _d}| | | j| |	\| _
| _|ttfv r|  j
d7  _
d| _qd| jS )NF   TrK   r   r0   )iterlencompatENCODINGrO   	ENDMARKERINDENTrU   DEDENTpoprM   rN   NEWLINENLrL   r[   r!   )r   iterableitindents	startliner<   tok_typetokenrV   endlineindentr   r   r   r      sF    



zUntokenizer.untokenizec           
      C   s   g }| j j}|d ttfv }d}t|g|D ]}|d d \}}	|tkrR|	| _q.|tt	fv rf|	d7 }	|t
kr|rzd|	 }	d}nd}|tkr||	 q.n>|tkr|  q.n*|ttfv rd}n|r|r||d  d}||	 q.d S )Nr   Fr\   rR   Tr]   )rL   rU   rf   rg   r5   chainra   rO   NAMENUMBERSTRINGrc   rd   re   )
r   rm   rh   rj   toks_appendrk   
prevstringtoktoknumtokvalr   r   r   r`      s8    
zUntokenizer.compatN)r   r   r   rP   r[   r   r`   r   r   r   r   rJ      s   %rJ   c                 C   s*   t  }|| }|jd ur&||j}|S r   )rJ   r   rO   encode)rh   utoutr   r   r   r     s
    

r   c                 C   sH   | d d   dd}|dks*|dr.dS |dv s@|drDdS | S )	N   _-utf-8zutf-8-)zlatin-1
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-r   )lowerreplace
startswith)orig_encencr   r   r   _get_normal_name  s    r   c                    s   zj jW n ty"   d Y n0 d d }d}fdd} fdd}| }|trnd |dd  }d	}|sz|g fS ||}|r||gfS t|s||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFr   c                      s"   z  W S  t y   Y dS 0 d S Nr%   )StopIterationr   readliner   r   read_or_stop?  s    z%detect_encoding.<locals>.read_or_stopc                    s   z|  d}W n2 ty@   d}d ur4d|}t|Y n0 t|}|sTd S t|d}zt|}W n8 t	y   d u rd| }nd|}t|Y n0  r|dkrԈd u rd}n
d}t||d	7 }|S )
Nr   z'invalid or missing encoding declarationz{} for {!r}rK   zunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrT   SyntaxError	cookie_rematchr   r$   r   LookupError)ro   line_stringmsgr   rO   codec)	bom_foundfilenamer   r   find_cookieE  s8    


z$detect_encoding.<locals>.find_cookieT   	utf-8-sig)__self__nameAttributeErrorr   r   blank_rer   )r   rO   defaultr   r   firstsecondr   )r   r   r   r   r
   '  s8    
&




r
   c                 C   sX   t | d}z2t|j\}}|d t||dd}d|_|W S    |   Y n0 d S )Nrbr   T)line_bufferingr+   )_builtin_openr
   r   seekr   modeclose)r   bufferrO   linestextr   r   r   r     s    

r   c                 C   s6   t | \}}td}t|t| d|}t|j|S r   )r
   r5   repeatrq   r^   	_tokenize__next__)r   rO   consumedemptyrl_genr   r   r   r     s    
r   c                 c   sz  d } }}d}d\}}d }dg}	|d urH|dkr6d}t t|dddV  d}
d}z|}
|  }W n tyt   d}Y n0 |d ur||}|d	7 }dt| }}|rn|std
|||}|r|d }}t t||d |  |||f|| V  d\}}d }nf|rX|dd  dkrX|dd  dkrXt t	|| ||t|f|V  d}d }qPn|| }|| }qPn|dkr|s|sqd}||k r|| dkr|d	7 }n8|| dkr|t
 d	 t
 }n|| dkrd}nq|d	7 }q||kr q|| dv r|| dkr\||d  d}t t|||f||t| f|V  |t|7 }t t||d  ||f|t|f|V  qP||	d kr|	| t t|d | |df||f|V  ||	d k r,||	vrtdd|||f|	d d }	t td||f||f|V  qn|s(td|dfd}||k rPtt||}|r|d	\}}||f||f|  }}}||kr~q,||| ||  }}||v s|dkr|dkr|dkrt t||||V  q|dv r|dkrt t||||V  nt t||||V  q|dkr0t t||||V  q|tv rtt| }|||}|r|d}||| }t t||||f|V  n||f}||d  }|}qPq|tv s|d d tv s|d d tv rD|d dkr0||f}tt|pt|d	 pt|d }||d  d	 }}|}qPnt t||||V  nf| rbt t||||V  nH|dkrrd	}n8|dv r|d	7 }n|d v r|d	8 }t t||||V  n*t t	|| ||f||d	 f|V  |d	7 }q,qP|
r.|
d dvr.|
 ds.t td|d	 t|
f|d	 t|
d	 fdV  |	d	d  D ] }t td|df|dfdV  q:t t d|df|dfdV  d S )!Nr   
0123456789)r0   r   r   r   )r   r   r0   r%   rK   zEOF in multi-line stringrQ   z\
rR   	z#
#z
r]   z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.z...r\   r   
\z([{z)]})!r   ra   r   r   r_   rG   r   rn   rt   
ERRORTOKENtabsizerstripCOMMENTrg   rU   rc   IndentationErrorrd   rB   PseudoTokenspanrs   rf   triple_quotedendpatssingle_quotedgetisidentifierrr   r   stripr   rb   )r   rO   lnumparenlev	continuednumcharscontstrneedcontcontlinerj   	last_linero   posmaxstrstartendprogendmatchrn   columncomment_tokenpseudomatchrV   sposeposrm   initialrp   r   r   r   r     s&   




*





"

 













$.r   c                 C   s
   t | d S r   )r   r   r   r   r   r	   d  s    r	   c               
      s@  dd l } dd  d fdd	}| jdd}|jdd	d
dd |jdddddd | }z|jr|j}t|d}tt|j}W d    q1 s0    Y  nd}t	t
jjd }|D ]>}|j}|jr|j}d|j|j  }	td|	t| |jf  qW n@ tyL }
 z8|
jd dd \}}||
jd |||f W Y d }
~
nd }
~
0  ty }
 z0|
jd \}}||
jd |||f W Y d }
~
nd }
~
0  ty }
 z||
| W Y d }
~
nd }
~
0  ty }
 z||
 W Y d }
~
nVd }
~
0  ty   td Y n4 ty: }
 z d|
   W Y d }
~
n
d }
~
0 0 d S )Nr   c                 S   s   t j|  t jd d S )Nr   )sysstderrwrite)messager   r   r   perrorp  s    zmain.<locals>.perrorc                    sR   |r"|f| | f } d|  n"|r8 d|| f  n d|   t d d S )Nz%s:%d:%d: error: %sz%s: error: %sz	error: %srK   )r   exit)r   r   locationargsr   r   r   errort  s    zmain.<locals>.errorzpython -m tokenize)progr   r(   zfilename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r   actionr   r   z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrK   r   zinterrupted
zunexpected error: %s)NN)argparseArgumentParseradd_argument
parse_argsr   r   listr   r   r   r   stdinr   r   r   rV   rn   printr   r   r   r   rG   r   OSErrorKeyboardInterrupt	Exception)r   r   parserr   r   r-   rL   rm   
token_typetoken_rangeerrro   r   r   r   r   mainl  sT    .**r   __main__)R
__author____credits__builtinsr   r   codecsr   r   collectionsior   	itertoolsr5   r>   r   rm   r   r?   ASCIIr   r   __all__
namedtupler   r$   r'   r)   
WhitespaceCommentIgnoreName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberr=   rB   StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStrPseudoExtrasr   r   _prefixsetr   r   r<   r,   r8   r   r   rG   rI   rJ   r   r   r
   r   r   r	   r   r   r   r   r   r   <module>   s   




_] 8=
