ó ŒwŪjc@sddlmZddlmZmZmZmZmZmZm Z m Z m Z mZddl m Z ddl Z ddlmZdefd„ƒYZd„Zd efd „ƒYZd efd „ƒYZd efd„ƒYZd„ZedkreƒndS(i’’’’(t test_support( t untokenizetgenerate_tokenstNUMBERtNAMEtOPtSTRINGt ENDMARKERttok_namet Untokenizerttokenize(tStringION(tTestCaset TokenizeTestcBs˜eZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z d„Z d „Z d „Z d „Z d „Zd „Zd„Zd„ZRS(c Cs‰g}t|ƒ}xTt|jƒD]C\}}}}} |tkrGPnt|}|jdtƒƒq"W|j||jƒj ƒƒdS(Ns4 %(type)-10.10s %(token)-13.13r %(start)s %(end)s( R RtreadlineRRtappendtlocalst assertEqualtrstript splitlines( tselftstexpectedtresulttfttypettokentstarttendtline((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pytcheck_tokenizes %    cCsc|jddƒ|jddƒd}|jtdƒ%xtt|ƒjƒD]}qOWWdQXdS(Ns1 + 1s… NUMBER '1' (1, 0) (1, 1) OP '+' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) s.if False: # NL True = False # NEWLINE s; NAME 'if' (1, 0) (1, 2) NAME 'False' (1, 3) (1, 8) OP ':' (1, 8) (1, 9) NEWLINE '\n' (1, 9) (1, 10) COMMENT '# NL' (2, 4) (2, 8) NL '\n' (2, 8) (2, 9) INDENT ' ' (3, 0) (3, 4) NAME 'True' (3, 4) (3, 8) OP '=' (3, 9) (3, 10) NAME 'False' (3, 11) (3, 16) COMMENT '# NEWLINE' (3, 17) (3, 26) NEWLINE '\n' (3, 26) (3, 27) DEDENT '' (4, 0) (4, 0) sdef k(x): x += 2 x += 5 s3unindent does not match any outer indentation level(RtassertRaisesRegexptIndentationErrorRR R(Rtindent_error_filettok((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_basics    cCs”|jddƒ|jddƒ|jddƒ|jddƒ|jd d ƒ|jd d ƒ|jd dƒ|jddƒ|jddƒdS(Ns 0xff <= 255s† NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) s 0b10 <= 255s† NUMBER '0b10' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) s 0o123 <= 0123s† NUMBER '0o123' (1, 0) (1, 5) OP '<=' (1, 6) (1, 8) NUMBER '0123' (1, 9) (1, 13) s01234567 > ~0x15sµ NUMBER '01234567' (1, 0) (1, 8) OP '>' (1, 9) (1, 10) OP '~' (1, 11) (1, 12) NUMBER '0x15' (1, 12) (1, 16) s2134568 != 01231515sˆ NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '01231515' (1, 11) (1, 19) s(-124561-1) & 0200000000sc OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '0200000000' (1, 14) (1, 24) s0xdeadbeef != -1s· NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) s0xdeadc0de & 012345sŠ NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '012345' (1, 13) (1, 19) s0xFF & 0x15 | 1234są NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_intBs$         cCsD|jddƒ|jddƒ|jddƒ|jddƒdS( Nsx = 0Ls… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0L' (1, 4) (1, 6) sx = 0xfffffffffffs† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xffffffffff (1, 4) (1, 17) sx = 123141242151251616110ls† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151 (1, 4) (1, 26) sx = -15921590215012591Ls± NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '159215902150 (1, 5) (1, 23) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_long{s    cCst|jddƒ|jddƒ|jddƒ|jddƒ|jd d ƒ|jd d ƒ|jd dƒdS(Ns x = 3.14159s† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) s x = 314159.s† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) s x = .314159s† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) s x = 3e14159s† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) s x = 3E123s… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) s x+y = 3e-1230sÜ NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) s x = 3.14e159s† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_float“s       cCs“|jddƒ|jddƒ|jddƒ|jddƒ|jd d ƒ|jd d ƒ|jd dƒ|jddƒ|jddƒ|jddƒ|jddƒdS(Nsx = ''; y = ""s5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) sx = '"'; y = "'"s6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) sx = "doesn't "shrink", does it"są NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) sx = u'abc' + U'ABC'są NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "u'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "U'ABC'" (1, 13) (1, 19) sy = u"ABC" + U"ABC"są NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'u"ABC"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'U"ABC"' (1, 13) (1, 19) s)x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC's” NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "ur'abc'" (1, 4) (1, 11) OP '+' (1, 12) (1, 13) STRING "Ur'ABC'" (1, 14) (1, 21) OP '+' (1, 22) (1, 23) STRING "uR'ABC'" (1, 24) (1, 31) OP '+' (1, 32) (1, 33) STRING "UR'ABC'" (1, 34) (1, 41) s)y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"s• NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'ur"abc"' (1, 4) (1, 11) OP '+' (1, 12) (1, 13) STRING 'Ur"ABC"' (1, 14) (1, 21) OP '+' (1, 22) (1, 23) STRING 'uR"ABC"' (1, 24) (1, 31) OP '+' (1, 32) (1, 33) STRING 'UR"ABC"' (1, 34) (1, 41) sb'abc' + B'abc's† STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) sb"abc" + B"abc"s† STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) s%br'abc' + bR'abc' + Br'abc' + BR'abc's; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) s%br"abc" + bR"abc" + Br"abc" + BR"abc"s; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_string»s,           cCs$|jddƒ|jddƒdS(Ns!def d22(a, b, c=2, d=2, *k): passs NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) sdef d01v_(a=1, *k, **w): passs¢ NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_functions  cCs|jdddƒdS(Ns*if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != s.1 and 5 in 1 not in 1 is 1 or 5 is not 1: passs› NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_comparisonAs !cCs|jddƒdS(Nsx = 1 << 1 >> 5s6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_shiftgs cCs|jddƒdS(Ns&x = 1 - y + 15 - 01 + 0x124 + z + a[5]s$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '01' (1, 17) (1, 19) OP '+' (1, 20) (1, 21) NUMBER '0x124' (1, 22) (1, 27) OP '+' (1, 28) (1, 29) NAME 'z' (1, 30) (1, 31) OP '+' (1, 32) (1, 33) NAME 'a' (1, 34) (1, 35) OP '[' (1, 35) (1, 36) NUMBER '5' (1, 36) (1, 37) OP ']' (1, 37) (1, 38) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_additivess cCs|jddƒdS(Nsx = 1//1*1/5*12%0x12s@ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_multiplicativeŠs cCs$|jddƒ|jddƒdS(Ns~1 ^ 1 & 1 |1 ^ -1sč OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) s-1*1/1+1*1//1 - ---1**1sH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_unaryœs  cCs|jddƒdS(Ns/import sys, time x = sys.modules['time'].time()sõ NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_selectorĮs cCs|jddƒdS(Ns @staticmethod def foo(x,y): passs OP '@' (1, 0) (1, 1) NAME 'staticmethod (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_methodŲs cCs|jddƒdS(Nsdef f(): if x passs_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_tabsźs cCs|jddƒdS(Ns @ s/ OP '@' (1, 0) (1, 1) (R(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt%test_pathological_trailing_whitespace’s (t__name__t __module__RR#R$R%R&R'R(R)R*R+R,R-R.R/R0R1(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyR s   # 9  ( ] ) &   %   cCsŖg}tt|ƒjƒ}x‚|D]z\}}}}}|tkr‰d|kr‰|jtdftdftt|ƒftdfgƒq"|j ||fƒq"Wt |ƒS(Nt.tDecimalt(t)( RR RRtextendRRRtreprRR(RRtgttoknumttokvalt_((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pytdecistmts  tTestMisccBseZd„ZRS(cCsnddlm}d}|jt|ƒdƒ|jtt|ƒƒdƒ|jtt|ƒƒ|dƒƒdS(Ni’’’’(R5s+21.3e-5*-.1234/81.7s8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')s-3.21716034272e-0+7s!-3.217160342717258261933904529E-7(tdecimalR5RR>tassertRegexpMatcheststrteval(RR5R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_decistmts (R2R3RD(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyR?stUntokenizeTestcBs#eZd„Zd„Zd„ZRS(cCsttƒ}d|_d|_|jtƒ}|jdƒWdQX|j|jjddƒ|jt|jdƒdS(Niiiis'start (1,3) precedes previous end (2,2)(ii(ii( R tprev_rowtprev_colt assertRaisest ValueErrortadd_whitespaceRt exceptiontargs(Rtutcm((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_bad_input_order-s   cCs}tƒ}d|_d|_g|_|jdƒ|j|jdgƒd|_|jd ƒ|j|jdddgƒdS( Niiis\ is\ \ s (ii(ii(R RFRGttokensRJR(RRM((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_backslash_continuation9s       cCsptƒ}tdf}|j|tgƒƒ|j|jdgƒtƒ}|j|jt|gƒƒdƒdS(NtHellosHello (R RtcompattiterRRPR(RRMR((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_iter_compatEs    (R2R3RORQRU(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyRE+s t TestRoundtripcBsPeZd„Zd„Zd„Zd„Zd„Zd„Zd„Zd„Z RS(cCsµt|tƒrt|ƒ}ntt|jƒƒ}|jƒg|D]}|d ^qD}t|ƒ}t|j dƒƒj }gt|ƒD]}|d ^q‹}|j ||ƒdS(s/ Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized, converted back to source code via tokenize.untokenize(), and tokenized again from the latter. The test fails if the second tokenization doesn't match the first. iiN( t isinstanceRBR tlistRRtcloseRRTRtnextR(RRt token_listR"ttokens1tnew_textRttokens2((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pytcheck_roundtripPs  #cCsØ|jdƒ|jdƒ|jdƒ|jdƒtjdtjdƒ}t|ƒ}|j|ƒWdQX|jdƒ|jdƒ|jd ƒ|jd ƒdS( Nsif x == 1: print(x) sif x == 1: print x s# This is a comment # This alsosif x == 1 : print x ttokenize_teststtxtsnif x == 1: # A comment by itself. print x # Comment here, too. # Another comment. after_if = True sOif (x # The comments need to go in the right place == 1): print 'x==1' s¾class Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent srtry: import somemodule except ImportError: # comment print 'Can not import' # comment2 else: print 'Loaded' (R_Rtfindfiletostextseptopen(RtfnR((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_roundtrip`s       cCs|jdƒdS(Nsna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] (R_(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_continuationscCs|jdƒ|jdƒdS(Ns)x=1+\ 1 # This is a comment\ # This also s# Comment \ x = 0(R_(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyRQšs cCs|jdƒdS(Ns'' ''(R_(R((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_string_concatenation£scCsćddl}ddl}tjdtjdƒ}tjj|ƒpJtj}|jtjj |dƒƒ}tj dƒs|j |dƒ}nxM|D]E}y)t |dƒ}|j |ƒWdQXWq–d|GH‚q–Xq–WdS( Ni’’’’R`Rastest*.pytcpui trbsRoundtrip failed for file %s(tglobtrandomRRbRcRdtpathtdirnametcurdirtjointis_resource_enabledtsampleReR_(RRlRmRfttempdirt testfilesttestfileR((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyttest_random_files§s  cCsIt|tƒr!|jdƒ}ntt|ƒjƒ}t|ƒjdƒS(Nsutf-8(RWRBtencodeRR RRtdecode(RtcodeRP((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt roundtrip¼scCs:d}|j|ƒjdƒ}|j|d|dƒdS(s– Ensure that although whitespace might be mutated in a roundtrip, the semantic meaning of the indentation remains consistent. sif False: x=3 x=3 s iiN(R{tsplitR(RRzt codelines((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt#test_indentation_semantics_retainedĀs( R2R3R_RgRhRQRiRwR{R~(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyRVNs  -   cCs8tjtƒtjtƒtjtƒtjtƒdS(N(Rt run_unittestR RERVR?(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pyt test_mainĢs   t__main__(ttestRR RRRRRRRRR R RctunittestR R R>R?RERVR€R2(((sˆ/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_tokenize.pytsF ’ž #~