a äze%ÿã@sNddlmZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m ZmZmZmZddlmZmZddlZddlmZmZddlmZmZddlZddlZdd„ZGd d „d eƒZGd d „d eƒZd d„Z Gdd„deƒZ!Gdd„deƒZ"Gdd„deƒZ#Gdd„deƒZ$Gdd„deƒZ%Gdd„deƒZ&Gdd„deƒZ'e(dkrJe )¡dS)é)Úsupport)ÚtokenizeÚ _tokenizeÚ untokenizeÚNUMBERÚNAMEÚOPÚSTRINGÚ ENDMARKERÚENCODINGÚtok_nameÚdetect_encodingÚopenÚ UntokenizerÚgenerate_tokensÚNEWLINE)ÚBytesIOÚStringION)ÚTestCaseÚmock)ÚVALID_UNDERSCORE_LITERALSÚINVALID_UNDERSCORE_LITERALSc Csˆg}t| ¡ƒ}|ddv}|D]b\}}}}} |tkr:q„|rT|tkrT|d|krTq t|}| d|d›d|d›d|›d|›¡q |S)Néÿÿÿÿz rú Z10ú Z13)ÚlenÚ splitlinesr rr Úappend) Ztoken_generatorZ source_stringÚresultZ num_linesZmissing_trailing_nlÚtypeÚtokenÚstartÚendÚline©r$ú(/usr/lib/python3.9/test/test_tokenize.pyÚstringify_tokens_from_sources  (r&c@s¬eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)S)*Ú TokenizeTestcCs<t| d¡ƒ}tt|jƒ|ƒ}| |dg| ¡ ¡¡dS)Núutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))rÚencoder&rÚreadlineÚ assertEqualÚrstripr©ÚselfÚsÚexpectedÚfrr$r$r%Úcheck_tokenize*s ÿÿzTokenizeTest.check_tokenizecCsDtd d¡ƒ}tt|jƒƒ}| |djt¡| |djt¡dS)NÚxr(éþÿÿÿr) rr)Úlistrr*r+rrr )r.r1Útokensr$r$r%Útest_implicit_newline4sz"TokenizeTest.test_implicit_newlinecCsd| dd¡| dd¡d}t|ƒj}| td¡t|ƒD]}q ~0x15z´ NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) z2134568 != 1231515zˆ NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) z(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) z0xdeadbeef != -1z· NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) z0xdeadc0de & 12345zŠ NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) z0xFF & 0x15 | 1234zà NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) ©r2©r.r$r$r%Útest_intas       zTokenizeTest.test_intcCs4| dd¡| dd¡| dd¡| dd¡dS) Nzx = 0z… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) zx = 0xfffffffffffzˆ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xfffffffffff' (1, 4) (1, 17) zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151251616110' (1, 4) (1, 25) zx = -15921590215012591z· NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '15921590215012591' (1, 5) (1, 22) r<r=r$r$r%Ú test_longšs   zTokenizeTest.test_longcCsX| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡dS)Nz x = 3.14159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) z x = 314159.z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) z x = .314159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) z x = 3e14159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) z x = 3E123z… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) z x+y = 3e-1230zÜ NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) z x = 3.14e159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) r<r=r$r$r%Ú test_float²s      zTokenizeTest.test_floatcCsJdd„}tD]}d|vrq | ||ƒ|¡q tD]}| ||ƒ|¡q0dS)NcSs<t| d¡ƒ}t|jƒD]\}}}}}|tkr|SqdS)Nr(z invalid token)rr)rr*r)r/r1Ztoktyper r!r"r#r$r$r%Ú number_tokenÛs  z;TokenizeTest.test_underscore_literals..number_tokenú()rr+rZassertNotEqual)r.rAZlitr$r$r%Útest_underscore_literalsÚsz%TokenizeTest.test_underscore_literalscCs<| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd ¡| d!d"¡| d#d$¡| d%d&¡| d'd(¡| d)d*¡| d+d,¡| d-d.¡| d/d0¡| d1d2¡| d3d4¡dS)5Nzx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) zx = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) zx = "doesn't "shrink", does it"zà NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) zx = 'abc' + 'ABC'zß NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) zy = "ABC" + "ABC"zß NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a” NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a” NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) zu'abc' + U'abc'z† STRING "u'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "U'abc'" (1, 9) (1, 15) zu"abc" + U"abc"z† STRING 'u"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'U"abc"' (1, 9) (1, 15) zb'abc' + B'abc'z† STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) zb"abc" + B"abc"z† STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) z%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) z%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "rB'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Rb'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "RB'abc'" (1, 30) (1, 37) z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'rB"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) z "a\ de\ fg"z3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3) zu"a\ de"z/ STRING 'u"a\\\nde"' (1, 0) (2, 3) zrb"a\ d"z/ STRING 'rb"a\\\nd"' (1, 0) (2, 2) z """a\ b"""z0 STRING '"""a\\\nb"""' (1, 0) (2, 4) z u"""a\ b"""z1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4) zrb"""a\ b\ c"""z7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4) zf"abc"z/ STRING 'f"abc"' (1, 0) (1, 6) z fR"a{b}c"z/ STRING 'fR"a{b}c"' (1, 0) (1, 9) z f"""abc"""z0 STRING 'f"""abc"""' (1, 0) (1, 10) z f"abc\ def"z1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4) z Rf"abc\ def"z2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4) r<r=r$r$r%Ú test_stringésT        þÿÿÿÿþ   ÿÿzTokenizeTest.test_stringcCs(| dd¡| dd¡| dd¡dS)Nz!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) zdef d01v_(a=1, *k, **w): passa¢ NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) z&def d23(a: str, b: int=3) -> int: passaø NAME 'def' (1, 0) (1, 3) NAME 'd23' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ':' (1, 9) (1, 10) NAME 'str' (1, 11) (1, 14) OP ',' (1, 14) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'int' (1, 19) (1, 22) OP '=' (1, 22) (1, 23) NUMBER '3' (1, 23) (1, 24) OP ')' (1, 24) (1, 25) OP '->' (1, 26) (1, 28) NAME 'int' (1, 29) (1, 32) OP ':' (1, 32) (1, 33) NAME 'pass' (1, 34) (1, 38) r<r=r$r$r%Ú test_functions  zTokenizeTest.test_functioncCs| dd¡dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa› NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) r<r=r$r$r%Útest_comparisonËsÿzTokenizeTest.test_comparisoncCs| dd¡dS)Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) r<r=r$r$r%Ú test_shiftñszTokenizeTest.test_shiftcCs| dd¡dS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) r<r=r$r$r%Ú test_additiveýszTokenizeTest.test_additivecCs| dd¡dS)Nzx = 1//1*1/5*12%0x12@42aš NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) OP '@' (1, 20) (1, 21) NUMBER '42' (1, 21) (1, 23) r<r=r$r$r%Útest_multiplicativesz TokenizeTest.test_multiplicativecCs| dd¡| dd¡dS)Nz~1 ^ 1 & 1 |1 ^ -1aè OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) z-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) r<r=r$r$r%Ú test_unary(s zTokenizeTest.test_unarycCs| dd¡dS)Nz/import sys, time x = sys.modules['time'].time()aõ NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) r<r=r$r$r%Ú test_selectorMszTokenizeTest.test_selectorcCs| dd¡dS)Nz @staticmethod def foo(x,y): passa OP '@' (1, 0) (1, 1) NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) r<r=r$r$r%Ú test_methodcszTokenizeTest.test_methodcCs| dd¡dS)Nzdef f(): if x passa_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) r<r=r$r$r%Ú test_tabstsþzTokenizeTest.test_tabscCs| dd¡dS)Nu!Örter = 'places' grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "'places'" (1, 8) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "'green'" (2, 7) (2, 14) r<r=r$r$r%Útest_non_ascii_identifiers‰sz'TokenizeTest.test_non_ascii_identifierscCs| dd¡dS)Nu#Örter = u'places' grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "u'places'" (1, 8) (1, 17) NEWLINE '\n' (1, 17) (1, 18) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) r<r=r$r$r%Ú test_unicode•szTokenizeTest.test_unicodecCs| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd ¡| d!d"¡| d#d$¡| d%d&¡| d'd(¡| d)d*¡dS)+Nz async = 1z… NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) za = (async = 1)a8 NAME 'a' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '(' (1, 4) (1, 5) NAME 'async' (1, 5) (1, 10) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) zasync()z… NAME 'async' (1, 0) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) zclass async(Bar):passa< NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP '(' (1, 11) (1, 12) NAME 'Bar' (1, 12) (1, 15) OP ')' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 17) (1, 21) zclass async:passzµ NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP ':' (1, 11) (1, 12) NAME 'pass' (1, 12) (1, 16) z await = 1z… NAME 'await' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) z foo.asyncz… NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) zasync for a in b: passa; NAME 'async' (1, 0) (1, 5) NAME 'for' (1, 6) (1, 9) NAME 'a' (1, 10) (1, 11) NAME 'in' (1, 12) (1, 14) NAME 'b' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 18) (1, 22) zasync with a as b: passa< NAME 'async' (1, 0) (1, 5) NAME 'with' (1, 6) (1, 10) NAME 'a' (1, 11) (1, 12) NAME 'as' (1, 13) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'pass' (1, 19) (1, 23) z async.fooz… NAME 'async' (1, 0) (1, 5) OP '.' (1, 5) (1, 6) NAME 'foo' (1, 6) (1, 9) Úasyncz/ NAME 'async' (1, 0) (1, 5) zasync #comment awaitzÛ NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) COMMENT '#comment' (2, 0) (2, 8) NL '\n' (2, 8) (2, 9) NAME 'await' (3, 0) (3, 5) zasync ... awaitzÛ NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) OP '...' (2, 0) (2, 3) NEWLINE '\n' (2, 3) (2, 4) NAME 'await' (3, 0) (3, 5) z async awaitz… NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) NAME 'await' (2, 0) (2, 5) z foo.async + 1zß NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '1' (1, 12) (1, 13) zasync def foo(): passa; NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NAME 'pass' (1, 17) (1, 21) zNasync def foo(): def foo(await): await = 1 if 1: await async += 1 a× NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\n' (3, 13) (3, 14) DEDENT '' (4, 2) (4, 2) NAME 'if' (4, 2) (4, 4) NUMBER '1' (4, 5) (4, 6) OP ':' (4, 6) (4, 7) NEWLINE '\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) NAME 'await' (5, 4) (5, 9) NEWLINE '\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) NAME 'async' (6, 0) (6, 5) OP '+=' (6, 6) (6, 8) NUMBER '1' (6, 9) (6, 10) NEWLINE '\n' (6, 10) (6, 11) z)async def foo(): async for i in 1: passaÉ NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) NUMBER '1' (2, 17) (2, 18) OP ':' (2, 18) (2, 19) NAME 'pass' (2, 20) (2, 24) DEDENT '' (3, 0) (3, 0) zasync def foo(async): awaitah NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) NAME 'await' (1, 22) (1, 27) z@def f(): def baz(): pass async def bar(): pass await = 2aÐ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) zFasync def f(): def baz(): pass async def bar(): pass await = 2a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) OP ')' (1, 12) (1, 13) OP ':' (1, 13) (1, 14) NEWLINE '\n' (1, 14) (1, 15) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) r<r=r$r$r%Ú test_async¡s:            ù,þ ú%úzTokenizeTest.test_asyncN)Ú__name__Ú __module__Ú __qualname__r2r7r;r>r?r@rCrDrErFrGrHrIrJrKrLrMrNrOrQr$r$r$r%r'#s* %9('<& %  r'c@seZdZdd„ZdS)ÚGenerateTokensTestcCs0t|ƒ}tt|jƒ|ƒ}| || ¡ ¡¡dS©N)rr&rr*r+r,rr-r$r$r%r2´sz!GenerateTokensTest.check_tokenizeN)rRrSrTr2r$r$r$r%rU³srUcCs€g}tt| d¡ƒjƒ}|D]T\}}}}}|tkrbd|vrb| tdftdftt |ƒftdfg¡q|  ||f¡qt |ƒ  d¡S)Nr(Ú.ÚDecimalrBú)) rrr)r*rÚextendrrr ÚreprrrÚdecode)r/rÚgÚtoknumÚtokvalÚ_r$r$r%Údecistmt¼s ürac@seZdZdd„ZdS)ÚTestMisccCsPddlm}d}| t|ƒd¡| tt|ƒƒd¡| tt|ƒƒ|dƒ¡dS)Nr)rXz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)ÚdecimalrXr+raZ assertRegexr[Úeval)r.rXr/r$r$r%Ú test_decistmtÍs  ÿÿzTestMisc.test_decistmtN)rRrSrTrer$r$r$r%rbËsrbc@s<eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd S)ÚTestTokenizerAdheresToPep0263cCs,tj tj t¡|¡}t |t|dƒ¡dS)NÚrb)ÚosÚpathÚjoinÚdirnameÚ__file__Ú TestRoundtripÚcheck_roundtripr)r.Úfilenamerir$r$r%Ú _testFileçsz'TestTokenizerAdheresToPep0263._testFilecCsd}| |¡dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt©rp©r.r1r$r$r%Ú'test_utf8_coding_cookie_and_no_utf8_bomëszETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCsd}| t|j|¡dS)Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt©Ú assertRaisesÚ SyntaxErrorrprrr$r$r%Ú&test_latin1_coding_cookie_and_utf8_bomïszDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCsd}| |¡dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrqrrr$r$r%Ú"test_no_coding_cookie_and_utf8_bomúsz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCsd}| |¡dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrqrrr$r$r%Ú$test_utf8_coding_cookie_and_utf8_bomþszBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs$| t|jd¡| t|jd¡dS)Nz bad_coding.pyzbad_coding2.pyrtr=r$r$r%Útest_bad_coding_cookiesz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN) rRrSrTrprsrwrxryrzr$r$r$r%rfâs  rfc@seZdZdd„Zdd„ZdS)Ú Test_TokenizecsPd}| d¡‰d‰‡‡fdd„}tt|ddƒdd…}d g}| ||d ¡dS) Nõ "ЉЊЈÐЂ"r(Fcsˆs d‰ˆSdSdS©NTór$r$©Úfirstr#r$r%r* szNTest_Tokenize.test__tokenize_decodes_with_specified_encoding..readline©Úencodingér4©ér|)rƒr)rƒér|zbytes not decoded with encoding)r)r5rr+)r.Úliteralr*r6Úexpected_tokensr$rr%Ú.test__tokenize_decodes_with_specified_encoding s  ÿz.readlinerr4r„z*string not tokenized when encoding is None)r5rr+)r.r*r6rˆr$rŠr%Ú1test__tokenize_does_not_decode_with_encoding_nones ÿz?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rRrSrTr‰r‹r$r$r$r%r{sr{c@s¬eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)S)*ÚTestDetectEncodingcsd‰‡‡fdd„}|S)Nrcs$ˆtˆƒkrt‚ˆˆ}ˆd7‰|S©Nrƒ)rÚ StopIteration)r#©ÚindexÚlinesr$r%r*1s  z1TestDetectEncoding.get_readline..readliner$©r.r‘r*r$rr%Ú get_readline/szTestDetectEncoding.get_readlinecCs>d}t| |¡ƒ\}}| |d¡| |t|dd…ƒ¡dS)N)ó # something óprint(something) ódo_something(else) r(é)r r“r+r5©r.r‘r‚Úconsumed_linesr$r$r%Útest_no_bom_no_encoding_cookie:s z1TestDetectEncoding.test_no_bom_no_encoding_cookiecCs6d}t| |¡ƒ\}}| |d¡| |ddg¡dS)N)s# something r•r–ú utf-8-sigr”r•©r r“r+r˜r$r$r%Útest_bom_no_cookieDs  ÿz%TestDetectEncoding.test_bom_no_cookiecCs4d}t| |¡ƒ\}}| |d¡| |dg¡dS)N)ó# -*- coding: latin-1 -*- r•r–ú iso-8859-1ržrœr˜r$r$r%Útest_cookie_first_line_no_bomOs z0TestDetectEncoding.test_cookie_first_line_no_bomcCs4d}t| |¡ƒ\}}| |d¡| |dg¡dS)N)s# coding=utf-8 r•r–r›s# coding=utf-8 rœr˜r$r$r%Ú&test_matched_bom_and_cookie_first_lineYs z9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs d}| |¡}| tt|¡dS)N)s## vim: set fileencoding=ascii : r•r–©r“rurvr r’r$r$r%Ú.BunkcSs||_||_d|_dS)Nr)ÚnameZ_linesÚ_index)r.r‘rir$r$r%Ú__init__szDTestDetectEncoding.test_filename_in_exception..Bunk.__init__cs.|jtˆƒkrt‚ˆ|j}|jd7_|Sr)rÓrrŽ)r.r#©r‘r$r%r*s  zDTestDetectEncoding.test_filename_in_exception..Bunk.readlineN)rRrSrTrÔr*r$rÕr$r%ÚBunk srÖz.*{})rurvrÒr r*r8Úformat)r.rirÖZinsr$rÕr%Útest_filename_in_exceptions  ( z-TestDetectEncoding.test_filename_in_exceptioncCsTtdƒ}tjd|d| ttd¡Wdƒn1s:0Y| |j¡dS)Ns #coding:xxxztokenize._builtin_open)Z return_valueZfoobar)rrZpatchrurvrÎZ assertTrueÚclosed)r.Úmr$r$r%Útest_open_error#s,z"TestDetectEncoding.test_open_errorN)rRrSrTr“ršrr r¡r£r¨r«r¬r°r³rµrÀrÂrÃrÅrÇrÑrØrÛr$r$r$r%rŒ-s(              rŒc@s<eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd S)Ú TestTokenizecsžddl}tƒ‰d‰‡fdd„}‡fdd„}d‰‡fdd„}|j}|j}||_||_z,t|ƒ}| t|ƒgd¢¡W||_||_n||_||_0| ˆˆ¡dS) Nrcs ˆddgfS)Nófirstósecondr$)r*rr$r%Úmock_detect_encoding1sz8TestTokenize.test_tokenize..mock_detect_encodingcs"|‰g}|ƒ}|r| |¡q|SrV)r)r*r‚ÚoutZ next_line)Ú encoding_usedr$r%Úmock__tokenize4s z2TestTokenize.test_tokenize..mock__tokenizecs ˆd7‰ˆdkrdStˆƒ ¡S)Nrƒér~)Ústrr)r$)Úcounterr$r%Ú mock_readline@sz1TestTokenize.test_tokenize..mock_readline)rÝrÞó1ó2ó3ó4)rÚobjectr rr+r5)r.Ztokenize_modulerßrâræZorig_detect_encodingZorig__tokenizeÚresultsr$)rår‚rár%Ú test_tokenize-s*    ÿÿzTestTokenize.test_tokenizecCsfg}tdƒD]}| dj|d¡q | d¡d |¡}ttt| d¡ƒjƒƒ}|  |dj d¡dS)Niôzdef i{i}(): return {i})ÚiZOKÚ r(éýÿÿÿ) Úrangerr×rjr5rrr)r*r+Ústring)r.ÚbufrîZtoksr$r$r%Útest_oneline_defsUs   zTestTokenize.test_oneline_defscGsÂttt| d¡ƒjƒƒ}t|ƒ}| t|ƒd|¡| t|djtt ¡t |ƒD]&}| t||djt||¡qV| t|d|jtt j ¡| t|d|jtt j ¡dS)Nr(r…rrƒr—)r5rrr)r*rr+r Ú exact_typer rñr rr )r.ZopstrZoptypesr6Z num_optypesrîr$r$r%ÚassertExactTypeEqualas ÿ  ÿÿÿz!TestTokenize.assertExactTypeEqualcCsø| dtjtj¡| dtjtj¡| dtj¡| dtj¡| dtj¡| dtj ¡| dtj ¡| dtj ¡| d tj ¡| d tj ¡| d tj¡| d tj¡| d tj¡| dtj¡| dtj¡| dtj¡| dtjtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj ¡| dtj!¡| dtj"¡| dtj#¡| d tj$¡| d!tj%¡| d"tj&¡| d"tj&¡| d#tj'¡| d$tj(¡| d%tj)¡| d&tj*¡| d'tj+¡| d(tj,¡| d)tj-¡| d*tj.¡| d+tj/¡| d,tj0¡| d-t1tjt2tj t1tjt2tjt1tjt2¡ | d.tjtj2tjtj2tjtj2tj¡| d/tjtjtj1tjtj2tj¡dS)0Nz()z[]ú:ú,ú;ú+r·Ú*ú/ú|ú&ú<ú>ú=rWú%z{}z==z!=z<=z>=ú~ú^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->ú@z@=za**2+b**2==c**2z {1, 2, 3}z ^(x & 0x1))3rör ÚLPARÚRPARÚLSQBÚRSQBÚCOLONÚCOMMAÚSEMIÚPLUSÚMINUSÚSTARÚSLASHÚVBARÚAMPERÚLESSÚGREATERÚEQUALÚDOTÚPERCENTÚLBRACEÚRBRACEÚEQEQUALÚNOTEQUALÚ LESSEQUALÚ GREATEREQUALÚTILDEÚ CIRCUMFLEXÚ LEFTSHIFTÚ RIGHTSHIFTÚ DOUBLESTARÚ PLUSEQUALÚMINEQUALÚ STAREQUALÚ SLASHEQUALÚ PERCENTEQUALÚ AMPEREQUALÚ VBAREQUALÚCIRCUMFLEXEQUALÚLEFTSHIFTEQUALÚRIGHTSHIFTEQUALÚDOUBLESTAREQUALÚ DOUBLESLASHÚDOUBLESLASHEQUALÚ COLONEQUALÚELLIPSISÚRARROWÚATÚATEQUALrrr=r$r$r%Útest_exact_typeos‚ûû üzTestTokenize.test_exact_typecCs| dtj¡dS)Nz @ )rör r3r=r$r$r%Ú%test_pathological_trailing_whitespace°sz2TestTokenize.test_pathological_trailing_whitespacecCs¢d}tjtjtjtjtjtjg}ttt |  d¡ƒj ƒƒ}|  t |djt t¡tdƒD]&}|  t ||djt ||¡qZ|  t |djt tj¡dS)Nz b = 1 #testr(rérƒr)r rrrrÚNLÚCOMMENTr5rrr)r*r+r rõr rñr )r.Úsourcerˆr6rîr$r$r%Ú5test_comment_at_the_end_of_the_source_without_newline´s $zBTestTokenize.test_comment_at_the_end_of_the_source_without_newlineN) rRrSrTrírôrör5r6r;r$r$r$r%rÜ+s ( ArÜc@s$eZdZdd„Zdd„Zdd„ZdS)ÚUntokenizeTestcCsntƒ}d|_d|_| t¡}| d¡Wdƒn1s<0Y| |jjdd¡| t|jd¡dS)Nr—)rƒr…rz'start (1,3) precedes previous end (2,2))r—rƒ) rÚprev_rowÚprev_colruÚ ValueErrorÚadd_whitespacer+Ú exceptionÚargs)r.ÚuÚcmr$r$r%Útest_bad_input_orderÁs (ÿz#UntokenizeTest.test_bad_input_ordercCsdtƒ}d|_d|_g|_| d¡| |jdg¡d|_| d¡| |jgd¢¡t |d¡dS)Nrƒ)r—rú\ r—)érG)rFz\ \ rza b c \ c )rr=r>r6r@r+rmrn)r.rCr$r$r%Útest_backslash_continuationÍs  z*UntokenizeTest.test_backslash_continuationcCsštƒ}tdf}tdf|g}| |tgƒ¡| |jdg¡tƒ}| | t|gƒ¡d¡tƒ}| | t|ƒ¡d¡| |jd¡| tt|ƒƒd¡dS)NZHellor(zHello sHello ) rrr ÚcompatÚiterr+r6rr‚)r.rCr r6r$r$r%Útest_iter_compatÚs zUntokenizeTest.test_iter_compatN)rRrSrTrErHrKr$r$r$r%r<¿s  r<c@sLeZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dS)rmc CsÆt|tƒr| d¡}n| ¡}| ¡t|jddƒj}tt |ƒƒ}dd„|Dƒ}t |ƒ}t|jddƒj}dd„t |ƒDƒ}|  ||¡t |ƒ} t| jddƒj} dd„t | ƒDƒ} |  | |¡dS)Nr(T)ÚkeependscSsg|]}|dd…‘qS©Nr—r$©Ú.0r:r$r$r%Ú þr~z1TestRoundtrip.check_roundtrip..cSsg|]}|dd…‘qSrMr$rNr$r$r%rPr~cSsg|]}|dd…‘qSrMr$rNr$r$r%rPr~) Ú isinstancerär)ÚreadÚcloserJrÚ__next__r5rrr+) r.r1Úcoder*Ztokens5Ztokens2Z bytes_from2Z readline2Z tokens2_from2Z bytes_from5Z readline5Z tokens2_from5r$r$r%rnês    zTestRoundtrip.check_roundtripcCsˆ| d¡| d¡| d¡t d¡}t|dƒ}| |¡Wdƒn1sR0Y| d¡| d¡| d¡| d ¡dS) Nzif x == 1: print(x) z # This is a comment # This also zif x == 1 : print(x) útokenize_tests.txtrgzoif x == 1: # A comment by itself. print(x) # Comment here, too. # Another comment. after_if = True zPif (x # The comments need to go in the right place == 1): print('x==1') z¾class Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent zttry: import somemodule except ImportError: # comment print('Can not import' # comment2 )else: print('Loaded') )rnrÚfindfiler)r.Úfnr1r$r$r%Útest_roundtrip s     (   zTestRoundtrip.test_roundtripcCs| d¡dS)Nzna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] ©rnr=r$r$r%Útest_continuation1szTestRoundtrip.test_continuationcCs| d¡| d¡dS)Nz)x=1+\ 1 # This is a comment\ # This also z# Comment \ x = 0rZr=r$r$r%rH>s z)TestRoundtrip.test_backslash_continuationcCs| d¡dS)Nz'' ''rZr=r$r$r%Útest_string_concatenationGsz'TestRoundtrip.test_string_concatenationc Csddl}ddl}t d¡}tj |¡p*tj}| tj |  |¡d¡¡}|  tj |d¡¡dD]}|  tj |d¡|¡q^t  d¡s’|  |d¡}|D]z}tj d kr®td |ƒt|d ƒF}|j|d | |¡Wdƒn1sæ0YWdƒq–1s0Yq–dS) NrrVztest*.pyztest_unicode_identifiers.py)ÚbufferZbuiltinZfileioÚinspectrhÚplatformÚsysz test_%s.pyZcpué r—rrgrÉ)ÚglobÚrandomrrWrhrirkÚcurdirrjÚescapeÚremoveZis_resource_enabledÚsampleÚverboserÍrZsubTestrn)r.rbrcrXÚtempdirZ testfilesr1Ztestfiler$r$r%Útest_random_filesKs      zTestRoundtrip.test_random_filescCs,t|tƒr| d¡}ttt|ƒjƒƒ d¡S)Nr()rQrär)rrrr*r\)r.rUr$r$r%Ú roundtriphs  zTestRoundtrip.roundtripcCs6d}| |¡ d¡}| |d|d¡| |¡dS)Nzif False: x=3 x=3 rïrƒr—)rkÚsplitr+rn)r.rUZ codelinesr$r$r%Ú#test_indentation_semantics_retainedmsz1TestRoundtrip.test_indentation_semantics_retainedN) rRrSrTrnrYr[rHr\rjrkrmr$r$r$r%rmès '  rmÚ__main__)*Útestrrrrrrrr r r r r rrÎrrrÚiorrZunittestrrZtest.test_grammarrrrhr r&r'rUrarbrfr{rŒrÜr<rmrRÚmainr$r$r$r%Ús> D %&)