
    <e9                    R   d dl mZ d dlmZ d dlmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ d dlmZmZ d dlZd dlmZ d dlmZmZ d dlmZm Z  d dlmZ d d	l!m"Z"m#Z# d dl$Z$d dl%Z%d
 Z& G d de          Z' G d de'          Z(d Z) G d de          Z* G d de          Z+ G d de          Z, G d de          Z- G d de          Z. G d de          Z/ G d de          Z0 G d de          Z1 G d  d!ej                  Z2e3d"k    r ej4                     dS dS )#    )support)	os_helper)tokenize	_tokenize
untokenizeNUMBERNAMEOPSTRING	ENDMARKERENCODINGtok_namedetect_encodingopenUntokenizergenerate_tokensNEWLINE!_generate_tokens_from_c_tokenizerDEDENT)BytesIOStringION)dedent)TestCasemock)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALS)run_test_scriptmake_scriptc                 $   g }t          |                                          }|d         dv}| D ]_\  }}}}}	|t          k    r nK|r|t          k    r|d         |k    r/t          |         }|                    d|dd|dd| d|            `|S )Nz
r       10 13)len
splitlinesr   r   r   append)
token_generatorsource_stringresult	num_linesmissing_trailing_nltypetokenstartendlines
             )/usr/lib/python3.11/test/test_tokenize.pystringify_tokens_from_sourcer3      s    FM,,..//I'+69)8 C C%eUC9E 	47??s1v7J7J~ATAAAuAAAEAACAABBBBM    c                       e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )TokenizeTestc                 
   t          |                    d                    }t          t          |j                  |          }|                     |dg|                                                                z              d S )Nutf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r   encoder3   r   readlineassertEqualrstripr&   selfsexpectedfr*   s        r2   check_tokenizezTokenizeTest.check_tokenize-   s~     AHHW%%&&-hqz.B.BAFFFG!**55778	9 	9 	9 	9 	9r4   c                 $   t          d                    d                    }t          t          |j                            }|                     |d         j        t                     |                     |d         j        t                     d S )Nxr8   r    )	r   r9   listr   r:   r;   r-   r   r   )r>   rA   tokenss      r2   test_implicit_newlinez"TokenizeTest.test_implicit_newline6   sr     CJJw''((hqz**++'222)44444r4   c                    |                      dd           |                      dd           d}t          |          j        }|                     t          d          5  t          |          D ]}	 d d d            d S # 1 swxY w Y   d S )Nz1 + 1z    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af      NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    s   def k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)rB   r   r:   assertRaisesRegexIndentationErrorr   )r>   indent_error_filer:   toks       r2   
test_basiczTokenizeTest.test_basic>   s   G & 		 		 		
 	 ;=		 		 		&
 ,--6##$4%>? ? 	 	  ))  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   A??BBc                    |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           d S )N0xff <= 255z    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0b10 <= 255z    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0o123 <= 0O123z    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    1234567 > ~0x15z    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    2134568 != 1231515z    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    (-124561-1) & 200000000ac      OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    0xdeadbeef != -1z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    0xdeadc0de & 12345z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    0xFF & 0x15 | 1234z    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rB   r>   s    r2   test_intzTokenizeTest.test_intc   s:   M , 		 		 		
 	M , 		 		 		
 	, / 		 		 		
 	- 0 		 		 		 	0 3 		 		 		
 	5 	8 			 			 			 	. 1 		 		 		 	0 3 		 		 		
 	0 3 		 		 		 		 		r4   c                     |                      dd           |                      dd           |                      dd           |                      dd           d S )	Nzx = 0z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    rY   rZ   s    r2   	test_longzTokenizeTest.test_long   s    G & 		 		 		
 	/ 2 		 		 		
 	7 : 		 		 		
 	4 7 		 		 		 		 		r4   c                 :   |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           d S )Nx = 3.14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    x = 314159.z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    x = .314159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    x = 3e14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    	x = 3E123z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    x+y = 3e-1230z    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    x = 3.14e159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rY   rZ   s    r2   
test_floatzTokenizeTest.test_float   s    M , 		 		 		
 	M , 		 		 		
 	M , 		 		 		
 	M , 		 		 		
 	K * 		 		 		
 	O . 		 		 		 	N - 		 		 		 		 		r4   c                     d }t           D ]&}d|v r|                      ||          |           't          D ]!}|                      ||          |           "d S )Nc                     t          |                     d                    }t          |j                  D ]\  }}}}}|t          k    r|c S dS )Nr8   zinvalid token)r   r9   r   r:   r   )r?   rA   toktyper.   r/   r0   r1   s          r2   number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_token   s]    ))**A4<QZ4H4H ! !0sDf$$ LLL %"?r4   ()r   r;   r   assertNotEqual)r>   rj   lits      r2   test_underscore_literalsz%TokenizeTest.test_underscore_literals   s    	# 	# 	# - 	5 	5Cczz\\#..4444. 	8 	8CS 1 137777	8 	8r4   c                 ~   |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd            |                      d!d"           |                      d#d$           |                      d%d&           |                      d'd(           |                      d)d*           |                      d+d,           |                      d-d.           |                      d/d0           |                      d1d2           |                      d3d4           d S )5Nx = ''; y = ""a5      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    x = '"'; y = "'"a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    x = "doesn't "shrink", does it"z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    x = 'abc' + 'ABC'z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    y = "ABC" + "ABC"z    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    %x = r'abc' + r'ABC' + R'ABC' + R'ABC'a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    %y = r"abc" + r"ABC" + R"ABC" + R"ABC"a      NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    u'abc' + U'abc'z    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    u"abc" + U"abc"z    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    b'abc' + B'abc'z    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    b"abc" + B"abc"z    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    %br'abc' + bR'abc' + Br'abc' + BR'abc'a;      STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    %br"abc" + bR"abc" + Br"abc" + BR"abc"a;      STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    %rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    %rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    "a\
de\
fg"3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    u"a\
de"/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    rb"a\
d"/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    
"""a\
b"""0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    u"""a\
b"""1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    rb"""a\
b\
c"""7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    f"abc"/    STRING     'f"abc"'      (1, 0) (1, 6)
    	fR"a{b}c"/    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
    
f"""abc"""0    STRING     'f"""abc"""'  (1, 0) (1, 10)
    f"abc\
def"1    STRING     'f"abc\\\ndef"' (1, 0) (2, 4)
    Rf"abc\
def"2    STRING     'Rf"abc\\\ndef"' (1, 0) (2, 4)
    rY   rZ   s    r2   test_stringzTokenizeTest.test_string   s   . 1 		 		 		 	1 4 		 		 		 	A D 		 		 		 	/ 2 		 		 		 	/ 2 		 		 		 	C 
F 
		 
		 
		 	C 
F 
		 
		 
		 	- 0 		 		 		
 	- 0 		 		 		 	- 0 		 		 		
 	- 0 		 		 		
 	C F 		 		 		 	C F 		 		 		 	C F 		 		 		 	C F 		 		 		 	 		 		 		
 	 		 		 		 	 		 		 		 	 		 		 		 	 		 		 		 	 		 		 		
 	H ' 		 		 		 	K * 		 		 		 	L + 		 		 		 	 		 		 		 	 		 		 		 		 		r4   c                     |                      dd           |                      dd           |                      dd           d S )N!def d22(a, b, c=2, d=2, *k): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    def d01v_(a=1, *k, **w): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    &def d23(a: str, b: int=3) -> int: passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rY   rZ   s    r2   test_functionzTokenizeTest.test_function  sr    ? B 		 		 		, 	; > 		 		 		" 	D G 		 		 		 		 		r4   c                 2    |                      dd           d S )NXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rY   rZ   s    r2   test_comparisonzTokenizeTest.test_comparison  0     M!O"		 "		 "		 "		 "		r4   c                 2    |                      dd           d S )Nzx = 1 << 1 >> 5a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    rY   rZ   s    r2   
test_shiftzTokenizeTest.test_shift  s,    - 0 		 		 		 		 		r4   c                 2    |                      dd           d S )N%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    rY   rZ   s    r2   test_additivezTokenizeTest.test_additive  -    C F 		 		 		 		 		r4   c                 2    |                      dd           d S )Nx = 1//1*1/5*12%0x12@42a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rY   rZ   s    r2   test_multiplicativez TokenizeTest.test_multiplicative  ,    5 8 		 		 		 		 		r4   c                 ^    |                      dd           |                      dd           d S )N~1 ^ 1 & 1 |1 ^ -1a      OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    -1*1/1+1*1//1 - ---1**1aH      OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rY   rZ   s    r2   
test_unaryzTokenizeTest.test_unary*  sN    0 3 		 		 		 	5 8 		 		 		 		 		r4   c                 2    |                      dd           d S )N/import sys, time
x = sys.modules['time'].time()a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    rY   rZ   s    r2   test_selectorzTokenizeTest.test_selectorO  -    N Q 		 		 		 		 		r4   c                 2    |                      dd           d S )N @staticmethod
def foo(x,y): passa      OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rY   rZ   s    r2   test_methodzTokenizeTest.test_methode  -    ? B 		 		 		 		 		r4   c                 2    |                      dd           d S )Nzdef f():
	if x
        	passa_      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    rY   rZ   s    r2   	test_tabszTokenizeTest.test_tabsv  s.     -/		 		 		 		 		r4   c                 2    |                      dd           d S )Nu!   Örter = 'places'
grün = 'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    rY   rZ   s    r2   test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiers  s-    @ C 		 		 		 		 		r4   c                 2    |                      dd           d S )N#   Örter = u'places'
grün = U'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rY   rZ   s    r2   test_unicodezTokenizeTest.test_unicode  -    B E 		 		 		 		 		r4   c                    |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd            |                      d!d"           |                      d#d$           |                      d%d&           |                      d'd(           |                      d)d*           d S )+N	async = 1z    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    a = (async = 1)a8      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    async()z    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    class async(Bar):passa<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    class async:passz    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    	await = 1z    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    	foo.asyncz    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    async for a in b: passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    async with a as b: passa<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    	async.fooz    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    asyncz/    NAME       'async'       (1, 0) (1, 5)
    async
#comment
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    async
...
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    async
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    foo.async + 1z    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    async def foo(): passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    )async def foo():
  async for i in 1: passa      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    async def foo(async): awaitah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    @def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    rY   rZ   s    r2   
test_asynczTokenizeTest.test_async  s   K * 		 		 		 	- 0 		 		 		 	I ( 		 		 		 	3 6 		 		 		 	. 1 		 		 		 	K * 		 		 		 	K * 		 		 		 	4 7 		 		 		 	5 8 		 		 		 	K * 		 		 		 	G & 		 		 		 	4 7 		 		 		 	/ 2 		 		 		 	N - 		 		 		 	O . 		 		 		 	3 6 		 		 		 	 #*		 *		 *		X 	 		 		 		* 	= 	@ 			 			 			 	 #		 #		 #		J 	 $		 $		 $		 $		 $		r4   N)__name__
__module____qualname__rB   rH   rN   r[   r]   rf   rn   r   r   r   r   r   r   r   r   r   r   r   r   r    r4   r2   r6   r6   &   sO       9 9 95 5 5# # #J7	 7	 7	r	 	 	0&	 &	 &	P8 8 8d	 d	 d	L:	 :	 :	x$	 $	 $	L
	 
	 
		 	 	.	 	 	(#	 #	 #	J	 	 	,	 	 	"	 	 	*
	 
	 
	
	 
	 
	P	 P	 P	 P	 P	r4   r6   c                       e Zd Zd ZdS )GenerateTokensTestc                     t          |          }t          t          |j                  |          }|                     ||                                                                           d S N)r   r3   r   r:   r;   r<   r&   r=   s        r2   rB   z!GenerateTokensTest.check_tokenize  sZ     QKK-oaj.I.I1MM!2!2!=!=!?!?@@@@@r4   N)r   r   r   rB   r   r4   r2   r   r     s(        A A A A Ar4   r   c           	         g }t          t          |                     d                    j                  }|D ]q\  }}}}}|t          k    rGd|v rC|                    t          dft          dft          t          |          ft          dfg           Z|
                    ||f           rt          |                              d          S )Nr8   .Decimalrk   ))r   r   r9   r:   r   extendr	   r
   r   reprr'   r   decode)r?   r*   gtoknumtokval_s         r2   decistmtr     s    F'**++455A$% 	, 	,1aVvMMy!S	f&S		     MM66*++++f$$W---r4   c                       e Zd Zd ZdS )TestMiscc                 .   ddl m} d}|                     t          |          d           |                     t          t          |                    d           |                     t          t          |                     |d                     d S )Nr   )r   z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)decimalr   r;   r   assertRegexr   eval)r>   r   r?   s      r2   test_decistmtzTestMisc.test_decistmt  s     	$#####"!S	U 	U 	U 	d1gg(BCCC 	hqkk** !DEE	G 	G 	G 	G 	Gr4   N)r   r   r   r   r   r4   r2   r   r     s(        G G G G Gr4   r   c                   6    e Zd ZdZd Zd Zd Zd Zd Zd Z	dS )	TestTokenizerAdheresToPep0263zU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    c                     t           j                            t           j                            t                    |          }t
                              | t          |d                     d S )Nrb)ospathjoindirname__file__TestRoundtripcheck_roundtripr   )r>   filenamer   s      r2   	_testFilez'TestTokenizerAdheresToPep0263._testFile  sK    w||BGOOH55x@@%%dDt,<,<=====r4   c                 4    d}|                      |           d S )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtr   r>   rA   s     r2   'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom      Gqr4   c                 L    d}|                      t          | j        |           dS )a  
        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtNassertRaisesSyntaxErrorr   r   s     r2   &test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom  s*     G+t~q99999r4   c                 4    d}|                      |           d S )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr   r   s     r2   "test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom  r   r4   c                 4    d}|                      |           d S )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr   r   s     r2   $test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom   s    Dqr4   c                     |                      t          | j        d           |                      t          | j        d           d S )Nzbad_coding.pyzbad_coding2.pyr  rZ   s    r2   test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookie  s=    +t~GGG+t~7GHHHHHr4   N)
r   r   r   __doc__r   r   r  r  r	  r  r   r4   r2   r   r     s}         > > >  	: 	: 	:    I I I I Ir4   r   c                       e Zd Zd Zd ZdS )Test_Tokenizec                     d}|                     d          dfd}t          t          |d                    dd         }dg}|                     ||d	           d S )
N   "ЉЊЈЁЂ"r8   Fc                       sd S dS NTr4   r   )firstr1   s   r2   r:   zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline  s     sr4   encoding   rE      r  )r  r   )r     r  zbytes not decoded with encoding)r9   rF   r   r;   )r>   literalr:   rG   expected_tokensr  r1   s        @@r2   .test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encoding  s     ~~g&&	 	 	 	 	 	 i7;;;<<QrTBNO:	< 	< 	< 	< 	<r4   c                     ddfd}t          t          |d                     d d         }dg}|                     ||d           d S )Nr  Fc                       sd S dS r  r   )r  r  s   r2   r:   zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readline   s     sr4   r  rE   r  z*string not tokenized when encoding is None)rF   r   r;   )r>   r:   rG   r  r  r  s       @@r2   1test__tokenize_does_not_decode_with_encoding_nonez?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_none  s     	 	 	 	 	 	 i488899#2#>NOE	G 	G 	G 	G 	Gr4   N)r   r   r   r  r  r   r4   r2   r  r  	  s7        < < <$G G G G Gr4   r  c                       e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd ZdS )TestDetectEncodingc                     dfd}|S )Nr   c                  V    t                    k    rt                   } dz  | S Nr  )r%   StopIteration)r1   indexliness    r2   r:   z1TestDetectEncoding.get_readline.<locals>.readline3  s1    E

""##<DQJEKr4   r   )r>   r'  r:   r&  s    ` @r2   get_readlinezTestDetectEncoding.get_readline1  s.    	 	 	 	 	 	 r4   c                     d}t          |                     |                    \  }}|                     |d           |                     |t          |d d                              d S )N)   # something
   print(something)
   do_something(else)
r8      )r   r(  r;   rF   r>   r'  r  consumed_liness       r2   test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookie<  sk    

 $343D3DU3K3K#L#L .7+++eBQBi99999r4   c                     d}t          |                     |                    \  }}|                     |d           |                     |ddg           d S )N)s   ﻿# something
r+  r,  	utf-8-sigr*  r+  r   r(  r;   r.  s       r2   test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookieF  ss    

 $343D3DU3K3K#L#L .;///*,AB	D 	D 	D 	D 	Dr4   c                     d}t          |                     |                    \  }}|                     |d           |                     |dg           d S )N)   # -*- coding: latin-1 -*-
r+  r,  
iso-8859-1r6  r3  r.  s       r2   test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bomQ  sa    

 $343D3DU3K3K#L#L .<000*H)IJJJJJr4   c                     d}t          |                     |                    \  }}|                     |d           |                     |dg           d S )N)s   ﻿# coding=utf-8
r+  r,  r2  s   # coding=utf-8
r3  r.  s       r2   &test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_line[  sa    

 $343D3DU3K3K#L#L .;///*=)>?????r4   c                 v    d}|                      |          }|                     t          t          |           d S )N)s#   ﻿# vim: set fileencoding=ascii :
r+  r,  r(  r  r  r   r>   r'  r:   s      r2   <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrore  s<    

 $$U+++AAAAAr4   c                     d}t          |                     |                    \  }}|                     |d           ddg}|                     ||           d S )N)   #! something
    # vim: set fileencoding=ascii :
r+  r,  asciir@  rA  r3  r>   r'  r  r/  r@   s        r2   test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_bomn  sh    
 $343D3DU3K3K#L#L .7+++%'KL22222r4   c                     d}t          |                     |                    \  }}|                     |d           |                     |ddg           d S )N)   ﻿#! something
   f# coding=utf-8
r+  r,  r2  r@  rG  r3  r.  s       r2   'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_linez  ss    
 $343D3DU3K3K#L#L .;///+-AB	D 	D 	D 	D 	Dr4   c                 v    d}|                      |          }|                     t          t          |           d S )N)rF  rA  r+  r,  r<  r=  s      r2   =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror  s<    
 $$U+++AAAAAr4   c                     d}t          |                     |                    \  }}|                     |d           dg}|                     ||           d S )N)   print('£')
%   # vim: set fileencoding=iso8859-15 :
   print('€')
r8   rL  r3  rC  s        r2   /test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_line  se    

 $343D3DU3K3K#L#L .7+++*+22222r4   c                     d}t          |                     |                    \  }}|                     |d           ddg}|                     ||           d S )N)   #print('£')
rM  rN  
iso8859-15rQ  rM  r3  rC  s        r2   ,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_line  sh    

 $343D3DU3K3K#L#L .<000+-VW22222r4   c                     d}t          |                     |                    \  }}|                     |d           ddg}|                     ||           d S )N)   
rM  rN  rR  rU  rM  r3  rC  s        r2   (test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line  sh    

 $343D3DU3K3K#L#L .<000DE22222r4   c                    d}|D ]y}dD ]t}|                     d|          }dd|                    d          z   dz   dd	f}|                     |          }t          |          \  }}|                     |d
           uzd S )N)zlatin-1r7  ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac-r   rY     #!/usr/bin/python

   # coding: rB  rU  s   print(things)
s   do_something += 4
r7  replacer9   r(  r   r;   	r>   	encodingsr  repencr'  rlfoundr/  s	            r2   test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalization  s    ;	! 		6 		6H! 6 6&&sC00/&G)<)<<uD+/1 &&u--(7(;(;%~  55556		6 		6r4   c                 v    d}|                      |          }|                     t          t          |           d S )Ns
   print("")r<  r=  s      r2   test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1  s<     $$U+++AAAAAr4   c                     d}|D ]x}dD ]s}|                     d|          }dd|                    d          z   dz   df}|                     |          }t          |          \  }}|                     |d	           tyd S )
N)r8   z	utf-8-macz
utf-8-unixrX  rY  rZ  r[  rB  rU  s   1 + 3
r8   r\  r^  s	            r2   test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization  s    8	! 	1 	1H! 1 1&&sC00/&G)<)<<uD#% &&u--(7(;(;%~  00001	1 	1r4   c                 
   |                      d          }t          |          \  }}|                     |d           |                     |dg           t          |                      d                    \  }}|                     |d           |                     |g            |                      d          }t          |          \  }}|                     |d           |                     |dg           |                      d          }t          |          \  }}|                     |d           |                     |g            |                      d          }|                     t          t          |           d S )	N)r+  r8   r+  r   )s   ﻿print(something)
r2  )s   ﻿)s   # coding: bad
)r(  r   r;   r  r  r>   r:   r  r/  s       r2   test_short_filesz#TestDetectEncoding.test_short_files  s~   $$%=>>#28#<#< .7+++*?)@AAA#243D3DR3H3H#I#I .7+++,,,$$%IJJ#28#<#< .;///*?)@AAA$$%788#28#<#< .;///,,,$$%:;;+AAAAAr4   c                     |                      d          }t          |          \  }}|                     |d           |                     |dg           d S )N)   print("#coding=fake")r8   rn  )r(  r   r;   rk  s       r2   test_false_encodingz&TestDetectEncoding.test_false_encoding  s^    $$%@AA#28#<#< .7+++*B)CDDDDDr4   c                    t           j        dz   }|                     t           j        |           dD ]}t	          |d|          5 }t          d|z  |           t          d|           d d d            n# 1 swxY w Y   t          |          5 }|                     |j        |           |                     |j	        d           d d d            n# 1 swxY w Y   t	          |dd	          5 }t          d|           d d d            n# 1 swxY w Y   t          |          5 }|                     |j        d	           |                     |j	        d           d d d            d S # 1 swxY w Y   d S )
Nz.py)ziso-8859-15r8   wr  z# coding: %sfileu   print('euro:€')rr2  )
r   TESTFN
addCleanupunlinkr   printtokenize_openr;   r  mode)r>   r   r  fps       r2   	test_openzTestDetectEncoding.test_open  sy   #e+	((333 1 	/ 	/Hhh777 72nx/b9999,266667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 x(( /B  h777  #.../ / / / / / / / / / / / / / /
 (C+666 	3"(r2222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	38$$ 	+R[+666RWc***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+sH   &A88A<	?A<	7CC	C	3DDD+7E//E36E3c                    d}d G fdd          }|                      t                    5   ||          }|`t          |j                   d d d            n# 1 swxY w Y   |                     t          d                    |                    5   ||          }t          |j                   d d d            d S # 1 swxY w Y   d S )Nsome_file_pathrf  c                        e Zd Zd Z fdZdS );TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc                 0    || _         || _        d| _        d S )Nr   )name_lines_index)r>   r'  r   s      r2   __init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__  s     	#r4   c                     | j         t                    k    rt          | j                  }| xj         dz  c_         |S r$  )r  r%   r%  )r>   r1   r'  s     r2   r:   zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline  s=    ;#e**,,''T[)q r4   N)r   r   r   r  r:   )r'  s   r2   Bunkr    s=             
      r4   r  z.*{})r  r  r  r   r:   rJ   format)r>   r   r  insr'  s       @r2   test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exception	  s{   	 	 	 	 	 	 	 	 	 	 {++ 	* 	*$ud##CCL)))		* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*
 ##Kt1D1DEE 	* 	*$ud##CCL)))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*s#   #AA"%A"!CC	C	c                     t          d          }t          j        d|          5  |                     t          t
          d           d d d            n# 1 swxY w Y   |                     |j                   d S )Ns   #coding:xxxztokenize._builtin_open)return_valuefoobar)r   r   patchr  r  ry  
assertTrueclosed)r>   ms     r2   test_open_errorz"TestDetectEncoding.test_open_error%  s    N##Z0qAAA 	D 	Dk=(CCC	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D 	D!!!!!s   "AAAN)r   r   r   r(  r0  r4  r8  r:  r>  rD  rH  rJ  rO  rS  rV  rd  rg  ri  rl  ro  r|  r  r  r   r4   r2   r!  r!  /  s[       	 	 	: : :	D 	D 	DK K K@ @ @B B B
3 
3 
3
D 
D 
DB B B	3 	3 	3	3 	3 	3	3 	3 	36 6 6B B B1 1 1B B B.E E E+ + +(* * *8" " " " "r4   r!  c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )TestTokenizec                 h  	
 dd l }t                      	d 
	fd}
fd}dfd}|j        }|j        }||_        ||_        	 t          |          }|                     t          |          g d           ||_        ||_        n# ||_        ||_        w xY w|                     
	           d S )Nr   c                     ddgfS )N   first   secondr   )r:   r  s    r2   mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding3  s    h	222r4   c                 V    |g }	  |             }|r|                     |           #|S r   )r'   )r:   r  out	next_lineencoding_useds       r2   mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenize6  s?    $MC$HJJ	 JJy)))
r4   c                  `     dz    dk    rdS t                                                     S )Nr     r4   )strr9   )counters   r2   mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readlineB  s2    qLG!||sw<<&&(((r4   )r  r     1   2   3   4)r   objectr   r   r;   rF   )r>   tokenize_moduler  r  r  orig_detect_encodingorig__tokenizeresultsr  r  r  s           @@@r2   test_tokenizezTestTokenize.test_tokenize/  s$   ****88	3 	3 	3 	3 	3		 		 		 		 		 	) 	) 	) 	) 	)  />(2*>'$2!	7}--GT']]JJJL L L /CO+(6O%% /CO+(6O%666611111s   4B	 	Bc           	         g }t          d          D ]+}|                    d                    |                     ,|                    d           d                    |          }t	          t          t          |                    d                    j                            }| 	                    |d         j
        d           d S )Ni  zdef i{i}(): return {i})iOK
r8   )ranger'   r  r   rF   r   r   r9   r:   r;   string)r>   bufr  tokss       r2   test_oneline_defszTestTokenize.test_oneline_defsW  s    s 	= 	=AJJ/666;;<<<<

4iinn HWSZZ%8%899BCCDDb$/////r4   c           	         t          t          t          |                    d                    j                            }t          |          }|                     t          |          d|z              |                     t          |d         j                 t          t                              t          |          D ]B}|                     t          ||dz            j                 t          ||                             C|                     t          |d|z            j                 t          t          j                            |                     t          |d|z            j                 t          t          j                            d S )Nr8   r  r   r  r-  )rF   r   r   r9   r:   r%   r;   r   
exact_typer   r  r.   r   r   )r>   opstroptypesrG   num_optypesr  s         r2   assertExactTypeEqualz!TestTokenize.assertExactTypeEqualc  sM   hwu||G'<'<==FGGHH'llVa+o666&)"67!(+	- 	- 	-{## 	3 	3AXfQUm&>?%gaj13 3 3 3&["9"DE!%-0	2 	2 	2&["9"DE!%/2	4 	4 	4 	4 	4r4   c                    |                      dt          j        t          j                   |                      dt          j        t          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j	                   |                      dt          j
                   |                      dt          j                   |                      d	t          j                   |                      d
t          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j        t          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                   |                      dt          j                    |                      dt          j!                   |                      dt          j"                   |                      dt          j#                   |                      d t          j$                   |                      d!t          j%                   |                      d"t          j&                   |                      d"t          j&                   |                      d#t          j'                   |                      d$t          j(                   |                      d%t          j)                   |                      d&t          j*                   |                      d't          j+                   |                      d(t          j,                   |                      d)t          j-                   |                      d*t          j.                   |                      d+t          j/                   |                      d,t          j0                   |                      d-tb          t          j        td          t          j	        tb          t          j        td          t          j        tb          t          j        td                     |                      d.t          j        t          j2        t          j        t          j2        t          j        t          j2        t          j                   |                      d/t          j        t          j        t          j1        t          j        t          j2        t          j                   d S )0Nz()z[]:,;+rY  */|&<>=r   %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))3r  r.   LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL	LESSEQUALGREATEREQUALTILDE
CIRCUMFLEX	LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR	PLUSEQUALMINEQUAL	STAREQUAL
SLASHEQUALPERCENTEQUAL
AMPEREQUAL	VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUALDOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr	   r   rZ   s    r2   test_exact_typezTestTokenize.test_exact_typeq  s   !!$
EJ???!!$
EJ???!!#u{333!!#u{333!!#uz222!!#uz222!!#u{333!!#uz222!!#u{333!!#uz222!!#u{333!!#uz222!!#u}555!!#u{333!!#uy111!!#u}555!!$elCCC!!$666!!$777!!$888!!$(:;;;!!#u{333!!#u'7888!!$888!!$(8999!!$(8999!!$888!!$777!!$888!!$(8999!!$(:;;;!!$(8999!!$888!!$(=>>>!!$(=>>>!!%)=>>>!!%)>???!!%)>???!!$(9:::!!%)?@@@!!$(8999!!%888!!$555!!#ux000!!$666!!"3"&(8&"'*"&(8&"'-"&(8&	B 	B 	B 	!!+"',"',"',"',"',	0 	0 	0 	!!,"'"2"'*"'*ek5<"'*		. 	. 	. 	. 	.r4   c                 F    |                      dt          j                   d S )Nz@          )r  r.   r  rZ   s    r2   %test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespace  s     !!-:::::r4   c           	         d}t           j        t           j        t           j        t           j        t           j        t           j        g}t          t          t          |
                    d                    j                            }|                     t          |d         j                 t          t                              t!          d          D ]B}|                     t          ||dz            j                 t          ||                             C|                     t          |d         j                 t          t           j                            d S )Nzb = 1

#testr8   r      r  r    )r.   r	   r  r   r   NLCOMMENTrF   r   r   r9   r:   r;   r   r  r   r  r   )r>   sourcer  rG   r  s        r2   5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newline  s    ! :u{EL%-QVQY[`[hihwv}}W'='=>>GHHII&)"67(9KLLLq 	_ 	_AXfQUm&>?/Z[J\A]^^^^&*"78(5?:STTTTTr4   N)	r   r   r   r  r  r  r  r  r  r   r4   r2   r  r  -  su        &2 &2 &2P	0 	0 	04 4 4?. ?. ?.B; ; ;	U 	U 	U 	U 	Ur4   r  c                        e Zd Zd Zd Zd ZdS )UntokenizeTestc                 Z   t                      }d|_        d|_        |                     t                    5 }|                    d           d d d            n# 1 swxY w Y   |                     |j        j        d         d           |                     t          |j        d           d S )Nr-  )r  r  r   z'start (1,3) precedes previous end (2,2))r-  r  )	r   prev_rowprev_colr  
ValueErroradd_whitespacer;   	exceptionargs)r>   ucms      r2   test_bad_input_orderz#UntokenizeTest.test_bad_input_order  s    MM

z** 	$bU###	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$*1-9	; 	; 	; 	*a&6>>>>>s   AA Ac                 V   t                      }d|_        d|_        g |_        |                    d           |                     |j        dg           d|_        |                    d           |                     |j        g d           t                              | d           d S )Nr  )r-  r   \
r-  )   r  )r  z\
\
r!   za
  b
    c
  \
  c
)r   r	  r
  rG   r  r;   r   r   )r>   r  s     r2   test_backslash_continuationz*UntokenizeTest.test_backslash_continuation  s    MM

	   F8,,,
	   #?#?#?@@@%%d,HIIIIIr4   c                 p   t                      }t          df}t          df|g}|                    |t	          g                      |                     |j        dg           t                      }|                     |                    t	          |g                    d           t                      }|                     |                    t	          |                    d           |                     |j        d           |                     t          t	          |                    d           d S )NHellor8   zHello s   Hello )	r   r	   r   compatiterr;   rG   r   r  )r>   r  r.   rG   s       r2   test_iter_compatzUntokenizeTest.test_iter_compat  s    MMwW%u-	R!!!H:...MMdE7mm44h???MMd6ll33X>>>W---DLL119=====r4   N)r   r   r   r  r  r  r   r4   r2   r  r    sD        
? 
? 
?J J J> > > > >r4   r  c                   >    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	S )
r   c                    t          |t                    r|                    d          }n(|                                }|                                 t          |                    d                    j        }t          t          |                    }d |D             }t          |          }t          |                    d                    j        }d t          |          D             }|                     ||           t          |          }	t          |	                    d                    j        }
d t          |
          D             }|                     ||           dS )a)  
        Test roundtrip for `untokenize`. `f` is an open file or a string.
        The source code in f is tokenized to both 5- and 2-tuples.
        Both sequences are converted back to source code via
        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
        The test fails if the 3 pair tokenizations do not match.

        When untokenize bugs are fixed, untokenize with 5-tuples should
        reproduce code that does not contain a backslash continuation
        following spaces.  A proper test should test this.
        r8   T)keependsc                 "    g | ]}|d d         S Nr-  r   .0rM   s     r2   
<listcomp>z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>   s     ...s3rr7...r4   c                 "    g | ]}|d d         S r  r   r   s     r2   r"  z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>       @@@SRaR@@@r4   c                 "    g | ]}|d d         S r  r   r   s     r2   r"  z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>	  r$  r4   N)
isinstancer  r9   readcloser  r&   __next__rF   r   r   r;   )r>   rA   coder:   tokens5tokens2bytes_from2	readline2tokens2_from2bytes_from5	readline5tokens2_from5s               r2   r   zTestRoundtrip.check_roundtrip  sM    a 	88G$$DD6688DGGIII6677@x))**..g... ))///>>??H	@@HY,?,?@@@000 ))///>>??H	@@HY,?,?@@@00000r4   c                    |                      d           |                      d           |                      d           t          j        d          }t          |d          5 }|                      |           d d d            n# 1 swxY w Y   |                      d           |                      d           |                      d           |                      d	           d S )
Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
tokenize_tests.txtr   zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
zclass Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)r   r   findfiler   )r>   fnrA   s      r2   test_roundtripzTestRoundtrip.test_roundtrip  sj    	 . 	/ 	/ 	/ - 	. 	. 	. 	 , 	- 	- 	-233"d^^ 	$q  ###	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 1 	2 	2 	2
 	 3 	4 	4 	4 	 B 	C 	C 	C 	 9 	: 	: 	: 	: 	:s   $BB
B
c                 0    |                      d           d S )Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
r   rZ   s    r2   test_continuationzTestRoundtrip.test_continuation3  s)     	& 		' 		' 		' 		' 		'r4   c                 Z    |                      d           |                      d           d S )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r9  rZ   s    r2   r  z)TestRoundtrip.test_backslash_continuation@  sH     - 	. 	. 	. 	 % 	& 	& 	& 	& 	&r4   c                 0    |                      d           d S )Nz'' ''r9  rZ   s    r2   test_string_concatenationz'TestRoundtrip.test_string_concatenationI  s    W%%%%%r4   c                 p   dd l }dd l}t          j        d          }t          j                            |          pt          j        }|                     t          j                            |	                    |          d                    }|
                    t          j                            |d                     dD ]8}|
                    t          j                            |d          |z             9t          j        d          s|                    |d          }|D ]}t          j        d	k    rt          d
|           t          |d          5 }|                     |          5  |                     |           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d S )Nr   r4  ztest*.pyztest_unicode_identifiers.py)bufferbuiltinfileioinspectr   platformsysz
test_%s.pycpu
   r-  r   r   rr  )globrandomr   r5  r   r   r   curdirr   escaperemoveis_resource_enabledsampleverboserx  r   subTestr   )r>   rG  rH  r6  tempdir	testfilesrA   testfiles           r2   test_random_fileszTestRoundtrip.test_random_filesM  s    	233'//"%%2IIbgll4;;w+?+?LLMM	 	g/LMMNNNT 	F 	FARW\\'<@@1DEEEE*511 	5i44I! 	, 	,H!##j(+++h%% ,\\x\00 , ,((+++, , , , , , , , , , , , , , ,, , , , , , , , , , , , , , ,	, 	,s6   F*1FF*FF*FF**F.	1F.	c                     t          |t                    r|                    d          }t          t	          t          |          j                                                d          S )Nr8   )r&  r  r9   r   r   r   r:   r   )r>   r*  s     r2   	roundtripzTestRoundtrip.roundtripj  sR    dC   	(;;w''D(74==#9::;;BB7KKKr4   c                     d}|                      |                              d          }|                     |d         |d                    |                     |           dS )z
        Ensure that although whitespace might be mutated in a roundtrip,
        the semantic meaning of the indentation remains consistent.
        zif False:
	x=3
	x=3
r  r  r-  N)rU  splitr;   r   )r>   r*  	codeliness      r2   #test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retainedo  s`    
 +NN4((..t44	1y|444T"""""r4   N)r   r   r   r   r7  r:  r  r=  rS  rU  rY  r   r4   r2   r   r     s        1 1 1@%: %: %:N' ' '& & && & &, , ,:L L L
# # # # #r4   r   c                   t    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd ZdS )CTokenizeTestc                    |                      |          5  t          t          |          |          }|                     ||                                                                           d d d            d S # 1 swxY w Y   d S )N)r  )rO  r3   r   r;   r<   r&   )r>   r?   r@   r*   s       r2   rB   zCTokenizeTest.check_tokenize{  s     \\\## 	E 	E11!44a F VX__%6%6%A%A%C%CDDD		E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	Es   AA<<B B c                    |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           d S )NrP   z    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rQ   z    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rR   z    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    rS   z    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    rT   z    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rU   ac      LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    rV   z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rW   z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rX   z    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rY   rZ   s    r2   r[   zCTokenizeTest.test_int  s:   M , 		 		 		 	M , 		 		 		 	, / 		 		 		 	- 0 		 		 		 	0 3 		 		 		 	5 	8 			 			 			 	. 1 		 		 		 	0 3 		 		 		 	0 3 		 		 		 		 		r4   c                 :   |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           d S )Nr_   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    r`   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    ra   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    rb   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    rc   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    rd   z    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    re   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rY   rZ   s    r2   rf   zCTokenizeTest.test_float  s    M , 		 		 		 	M , 		 		 		 	M , 		 		 		 	M , 		 		 		 	K * 		 		 		 	O . 		 		 		 	N - 		 		 		 		 		r4   c                 ~   |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd            |                      d!d"           |                      d#d$           |                      d%d&           |                      d'd(           |                      d)d*           |                      d+d,           |                      d-d.           |                      d/d0           |                      d1d2           |                      d3d4           d S )5Nrp   a5      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    rq   a6      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    rr   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    rs   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    rt   z    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    ru   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    rv   a      NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    rw   z    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    rx   z    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    ry   z    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    rz   z    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r{   a;      STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r|   a;      STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r}   a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r~   a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rY   rZ   s    r2   r   zCTokenizeTest.test_string  s   . 1 		 		 		 	1 4 		 		 		 	> A 		 		 		 	/ 2 		 		 		 	/ 2 		 		 		 	C 
F 
		 
		 
		 	C 
F 
		 
		 
		 	- 0 		 		 		 	- 0 		 		 		 	- 0 		 		 		 	- 0 		 		 		 	C F 		 		 		 	C F 		 		 		 	C F 		 		 		 	C F 		 		 		 	- 0 		 		 		 	L + 		 		 		 	L + 		 		 		 	 		 		 		 	 		 		 		 	 		 		 		 	H ' 		 		 		 	K * 		 		 		 	L + 		 		 		 	 		 		 		
 	 		 		 		 		 		r4   c                     |                      dd           |                      dd           |                      dd           d S )Nr   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rY   rZ   s    r2   r   zCTokenizeTest.test_function  sr    ? B 		 		 		. 	; > 		 		 		$ 	D G 		 		 		 		 		r4   c                 2    |                      dd           d S )Nr   a      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rY   rZ   s    r2   r   zCTokenizeTest.test_comparison  r   r4   c                 2    |                      dd           d S )Nr   a$      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    rY   rZ   s    r2   r   zCTokenizeTest.test_additive  r   r4   c                 2    |                      dd           d S )Nr   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rY   rZ   s    r2   r   z!CTokenizeTest.test_multiplicative%  r   r4   c                 ^    |                      dd           |                      dd           d S )Nr   a      TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r   aI      MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rY   rZ   s    r2   r   zCTokenizeTest.test_unary9  sN    0 3 		 		 		 	5 8 		 		 		 		 		r4   c                 2    |                      dd           d S )Nr   a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    rY   rZ   s    r2   r   zCTokenizeTest.test_selector_  r   r4   c                 2    |                      dd           d S Nr   a      AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rY   rZ   s    r2   r   zCTokenizeTest.test_methodu  r   r4   c                 2    |                      dd           d S rg  rY   rZ   s    r2   r   zCTokenizeTest.test_tabs  r   r4   c                    |                      dd           |                      dd           |                      dd           |                      dd           |                      d	d
           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd           |                      dd            |                      d!d"           |                      d#d$           |                      d%d&           |                      d'd(           |                      d)d*           d S )+Nr   z    ASYNC      'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   a8      NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    ASYNC      'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r   a<      NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r   z    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r   z    AWAIT      'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    r   a;      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r   a<      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r   z/    ASYNC      'async'       (1, 0) (1, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (3, 0) (3, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    AWAIT      'await'       (3, 0) (3, 5)
    r   z    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (2, 0) (2, 5)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r   a;      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    AWAIT      'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    ASYNC      'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r   ah      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    r   a~      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r   a      ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    rY   rZ   s    r2   r   zCTokenizeTest.test_async  s   K * 		 		 		 	- 0 		 		 		 	I ( 		 		 		 	3 6 		 		 		 	. 1 		 		 		 	K * 		 		 		 	K * 		 		 		 	4 7 		 		 		 	5 8 		 		 		 	K * 		 		 		 	G & 		 		 		 	4 7 		 		 		 	/ 2 		 		 		 	N - 		 		 		 	O . 		 		 		 	3 6 		 		 		 	 #*		 *		 *		X 	H K 		 		 		& 	9 	< 			 			 			 	 !		 !		 !		F 	 "		 "		 "		 "		 "		r4   c                 2    |                      dd           d S )Nr   u7      NAME       'Örter'       (1, 0) (1, 6)
    EQUAL      '='           (1, 7) (1, 8)
    STRING     "u'places'"   (1, 9) (1, 18)
    NEWLINE    ''            (1, 18) (1, 18)
    NAME       'grün'        (2, 0) (2, 5)
    EQUAL      '='           (2, 6) (2, 7)
    STRING     "U'green'"    (2, 8) (2, 16)
    rY   rZ   s    r2   r   zCTokenizeTest.test_unicode	  r   r4   c                    d }|                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d	           |                      t          |d
           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           |                      t          |d           d S )Nc                 :    t          t          |                     S r   )rF   r   r  s    r2   
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens	  s    9&AABBBr4   z(1+2]z(1+2}z{1+2]1_z1.2_1e2_z1e+    u   €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0x012z'sdfsdfz'''sdfsdf''a  ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))])r  r  )r>   rn  s     r2   test_invalid_syntaxz!CTokenizeTest.test_invalid_syntax	  s   	C 	C 	C 	+z7;;;+z7;;;+z7;;;+z4888+z6:::+z6:::+z5999+z6:::+z5999+z6:::+z7;;;+z5999+z6:::+z4888+z6:::+z7;;;+z5999+z6:::+z4888+z6:::+z4888+z4888+z5999+z6:::+z6:::+z5999+z9===+z=AAA+z3HIII+z377777r4   c                    d}d } ||dz
            }t          t          |                    }|                     |d         j        t                     t          |dd            ||          }t          t          |                    }|                     |d         j        t                     |                     t          t
          |dd           d S )Nd   c                 t    d                     d t          |           D                       }|d| z  dz   z  }|S )N c              3   &   K   | ]}d |z  dz   V  dS )  z	if True:
Nr   )r!  rD   s     r2   	<genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>	  s+      OO1dQh,6OOOOOOr4   r  zpass
)r   r  )indentsr  s     r2   generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source	  s@    WWOOgOOOOOFdWnx//FMr4   r  r    z<string>exec)	rF   r   r;   r-   r   compiler   r  rK   )r>   	MAXINDENTr  validrG   invalids         r2   test_max_indentzCTokenizeTest.test_max_indent	  s    		 	 	
  	A..7>>??&111z6***!/),,7@@AA'222gw
F	
 	
 	
 	
 	
r4   c                 4   d }t          d          }|                     |d           t          d          }|                      ||           ||                     t          d          }|                     |d           t          d          }|                      ||           ||                     t          d          }|                     |d	           t          d
          }|                      ||           ||                     d S )Nc                 4    d t          |           D             S )Nc                     g | ]	^}}}||f
S r   r   )r!  kindr  r   s       r2   r"  zYCTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens.<locals>.<listcomp>	  s#    ggg'9fqT6Ngggr4   )r   rm  s    r2   rn  zECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens	  s     gg=^_e=f=fggggr4   z
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae      NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj      NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        )r   rB   r;   )r>   rn  r*  code_no_conts       r2   #test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation	  s   	h 	h 	h     	D # 	 	 	,     	D))::l+C+CDDD    	D # 	 	 	    
 	D))::l+C+CDDD    	D # 	 	 	"     	D))::l+C+CDDDDDr4   N)r   r   r   rB   r[   rf   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r   r4   r2   r[  r[  z  s.       E E E?	 ?	 ?	B,	 ,	 ,	\t	 t	 t	l=	 =	 =	~$	 $	 $	L	 	 	.	 	 	($	 $	 $	L	 	 	,	 	 	"	 	 	"H	 H	 H	T
	 
	 
	&8 &8 &8P
 
 
(`E `E `E `E `Er4   r[  c                       e Zd Zd ZdS )CTokenizerBufferTestsc                     dddz   dddz   }t          j                    5 }t          |d|          }t          |           d d d            d S # 1 swxY w Y   d S )Nz"        #coding: latin-1
        #ai'  z

        #i'  foo)r   temp_dirr   r   )r>   test_scriptr  	file_names       r2   !test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_bufferM
  s    
e)  e)  !! 	'X#He[AAII&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	's   !AAAN)r   r   r   r  r   r4   r2   r  r  L
  s#        
' 
' 
' 
' 
'r4   r  __main__)5testr   test.supportr   r   r   r   r   r	   r
   r   r   r   r   r   r   ry  r   r   r   r   r   ior   r   unittesttextwrapr   r   r   test.test_grammarr   r   test.support.script_helperr   r   r   r.   r3   r6   r   r   r   r   r  r!  r  r  r   r[  r  r   mainr   r4   r2   <module>r     s3         " " " " " "I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I I !                      # # # # # # # #< < < < < < < < " " " " " " C C C C C C C C 				    M	 M	 M	 M	 M	8 M	 M	 M	^A A A A A A A A. . .G G G G Gx G G G."I "I "I "I "IH "I "I "IJ#G #G #G #G #GH #G #G #GL{" {" {" {" {" {" {" {"|RU RU RU RU RU8 RU RU RUh&> &> &> &> &>X &> &> &>RM# M# M# M# M#H M# M# M#`OE OE OE OE OEH OE OE OEd' ' ' ' 'H- ' ' ' zHMOOOOO r4   