]l X @ s4 d Z d Z d d l m Z d d l m Z m Z d d l Z d d l m
Z
d d l m Z d d l
Z
d d l Z d d l Te
j d e
j Z e
j d
e
j Z d d l Z e j d d d
d d d d g Z [ e Z d e e <e d Z d e e <e d Z d e e <e d 7Z d e d e d e d e d e d e d e d e! d e" d e# d e$ d e% d! e& d" e' d# e( d$ e) d% e* d& e+ d' e, d( e- d) e. d* e/ d+ e0 d, e1 d- e2 d. e3 d/ e4 d0 e5 d1 e6 d2 e7 d3 e8 d4 e9 d5 e: d6 e; d7 e< d8 e= d9 e> d: e? d; e@ d< eA d= eB d> eC d? eD d@ eE i, ZF GdA d d e jG d dB ZH dC dD ZI dE dF ZJ dG dH ZK dI ZL dJ ZM eL eJ dK eL eK eM ZN dL ZO dM ZP dN ZQ dO ZR dP ZS eI eP eQ eR eS ZT dQ ZU eI dR dS eK eU ZV dT eU ZW eI eV eW ZX eI dU eX dV ZY eI eY eX eT ZZ dW Z[ dX Z\ dY Z] dZ Z^ d[ Z_ eI e[ d\ e[ d] Z` eI e[ d^ e[ d_ Za eI d` da db d* dc dd de d- Zb df Zc eI dg dh di Zd eI eb ec ed Ze eI eZ ee ea eO Zf eN ef Zg eI e[ dj eI dk dK e[ dl eI dm dK Zh eI dn eM e` Zi eL eI ei eZ ee eh eO Zj do dp Zk dk e\ dm e] d\ e^ d] e_ dq e^ dr e_ ds e^ dt e_ du e^ dv e_ dw e^ dx e_ dy e^ dz e_ d{ e^ d| e_ d} e^ d~ e_ d e^ d e_ d e^ d e_ d e^ d e_ d e^ d e_ d e^ d e_ d e^ d e_ d e^ d e_ d d d d d d d d d d d d i& Zl i Zm x d D] Zn en em en <qCWi Zo x d D] Zn en eo en <qdWd Zp Gd d d eq Zr Gd d d eq Zs Gd d d Zt d d Zu d d Zv d d
Zw d d Z d d Zx d d Zy d d Zz d d Z{ e| d k r0e{ d S)zKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)COMMENTtokenizedetect_encodingNL
untokenizeENCODING TokenInfo ()[]:,;+-r /|&<>=.%{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=@z@=c @ s. e Z d Z d d Z e d d Z d S)r c C s. d | j t | j f } d | j d | S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)type)r( tok_name_replace)selfannotated_type r- -/opt/alt/python35/lib64/python3.5/tokenize.py__repr__c s zTokenInfo.__repr__c C s4 | j t k r) | j t k r) t | j S| j Sd S)N)r( OPstringEXACT_TOKEN_TYPES)r+ r- r- r.
exact_typeh s zTokenInfo.exact_typeN)__name__
__module__ __qualname__r/ propertyr3 r- r- r- r. r b s ztype string start end linec G s d d j | d S)Nr r r )join)choicesr- r- r. groupo s r: c G s t | d S)Nr )r: )r9 r- r- r. anyp s r; c G s t | d S)N?)r: )r9 r- r- r. maybeq s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z
0[bB][01]+z 0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z
[0-9]+[jJ]z[jJ]z (?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'''"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Zc C s t j | t j S)N)recompileUNICODE)exprr- r- r. _compile s rF r'''r"""b'''b"""R'''R"""B'''B"""br'''br"""bR'''bR"""Br'''Br"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""u'''u"""U'''U"""rRbBuUr'r"R'R"b'b"B'B"br'br"Br'Br"bR'bR"BR'BR"rb'rb"rB'rB"Rb'Rb"RB'RB"u'u"U'U" c @ s e Z d Z d S)
TokenErrorN)r4 r5 r6 r- r- r- r. r s r c @ s e Z d Z d S)StopTokenizingN)r4 r5 r6 r- r- r- r. r s r c @ s@ e Z d Z d d Z d d Z d d Z d d Z d S)
Untokenizerc C s( g | _ d | _ d | _ d | _ d S)Nr r )tokensprev_rowprev_colencoding)r+ r- r- r. __init__ s zUntokenizer.__init__c C s | \ } } | | j k s9 | | j k r] | | j k r] t d j | | | j | j | | j } | r | j j d | d | _ | | j } | r | j j d | d S)Nz+start ({},{}) precedes previous end ({},{})z\
r )r r
ValueErrorformatr append)r+ startrowcol
row_offset
col_offsetr- r- r. add_whitespace s -
zUntokenizer.add_whitespacec C s t | } g } d } x}| D]u} t | d k rH | j | | P| \ } } } } }
| t k ru | | _ q | t k r P| t k r | j | q n | t k r | j | \ | _
| _ q nl | t t
f k r d } nQ | r;| r;| d } | d t | k r5| j j | t | | _ d } | j | | j j | | \ | _
| _ | t t
f k r | j
d 7_
d | _ q Wd j | j S)NFr Tr r )iterlencompatr
r ENDMARKERINDENTr DEDENTpopr r NEWLINEr r r r8 ) r+ iterableitindents startlinettok_typetokenr endlineindentr- r- r. r sF
zUntokenizer.untokenizec
C sJ g } | j j } | d t t f k } d } xt | g | D]} | d d \ } } | t k ru | | _ qA | t t t t
f k r | d 7} | t k r | r d | } d } n d } | t k r | j | qA nW | t
k r | j qA n; | t t f k rd } n | r8| r8| | d d } | | qA Wd S)Nr Fr r Tr r )r r r r r r
r NAMENUMBERASYNCAWAITSTRINGr r r )
r+ r r r toks_appendr
prevstringtoktoknumtokvalr- r- r. r s8
zUntokenizer.compatN)r4 r5 r6 r r r r r- r- r- r. r s
%r c C s= t } | j | } | j d k r9 | j | j } | S)N)r r r encode)r utoutr- r- r. r ? s
c C sd | d d j j d d } | d k s= | j d rA d S| d k s\ | j d
r` d S| S)N _r zutf-8zutf-8-latin-1
iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)r r r )r r r )lowerreplace
startswith)orig_encencr- r- r. _get_normal_nameZ s " r c s1 y j j Wn t k
r* d Yn Xd d } d } f d d } f d d } | } | j t r d | d d } d } | s | g f S| | } | r | | g f St j | s | | g f S| } | s | | g f S| | } | r!| | | g f S| | | g f S)
NFzutf-8c s( y SWn t k
r# d SYn Xd S)N )
StopIterationr- )readliner- r. read_or_stop} s
z%detect_encoding.<locals>.read_or_stopc s2 y | j d } WnB t k
rW d } d k rG d j | } t | Yn Xt j | } | sq d St | j d } y t | } WnI t k
r d k r d | } n d j | } t | Yn X r.| d k r$ d k r d } n d j } t | | d 7} | S)
Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorr SyntaxError cookie_rematchr r: r LookupError)r line_stringmsgr r codec) bom_foundfilenamer- r. find_cookie s6
z$detect_encoding.<locals>.find_cookieTr z utf-8-sig)__self__nameAttributeErrorr r blank_rer )r r defaultr r firstsecondr- )r r r r. r
e s8
&
c C ss t | d } yH t | j \ } } | j d t | | d d } d | _ | SWn | j Yn Xd S)Nrbr line_bufferingTrc )
_builtin_openr
r seekr modeclose)r bufferr linestextr- r- r. r s
r c C s_ d d l m } m } t | \ } } t | d } | d } t | | | | j | S)Nr )r repeatr ) itertoolsr r r
r _tokenize__next__)r r r r consumedrl_genemptyr- r- r. r s
c c s d } } } d } d! \ } } d } d g } d }
d } d } d }
| d k r| | d k re d } t t | d" d# d Vxy
| } Wn t k
r d } Yn X| d k r | j | } | d 7} d t | } } | r| s t d | | j | } | rf| j d } } t t | | d | | | | f | | Vd$ \ } } d } q| r| d% d d k r| d& d d
k rt t | | | | t | f | Vd } d } q q| | } | | } q n| d k r| r| sPd } xv | | k r| | d k r7| d 7} n? | | d k r\| t
d t
} n | | d k rud } n P| d 7} qW| | k rP| | d k r| | d k r=| | d j d } | t | } t t | | | f | | t | f | Vt t
| | d | | f | t | f | Vq t t
t f | | d k | | d | | f | t | f | Vq | | d' k r| j | t t | d | | d f | | f | Vx | | d( k ri| | k r t d d | | | f | d d) } | rC| | d* k rCd } d }
d } t t d | | f | | f | VqW| r|
r| | d+ k rd } d }
d } n! | st d | d f d } xo| | k r- t t j | | } | r| j d \ } } | | f | | f | } } } | | k r-q| | | | | } } | | k sx| d k r| d k r| d k rt t | | | | Vq* | d k r|
r|
Vd }
| d k rt t
| | | | Vqt t | | | | V| rd }
q* | d k r2|
r|
Vd }
t t | | | | Vq* | t k rt t | } | j | | } | r| j d } | | | } t t | | | | f | Vq| | f } | | d } | } Pq* | t k s| d d
t k s| d d t k r| d, d k rl| | f } t t | pEt | d pEt | d
} | | d d } } | } Pqt t | | | | Vq* | j r| d- k r| rt | d k rt n t | | | | Vqt t | | | | } | d k r|
r| }
q| d k rm|
rm|
j t k rm|
j d k rmd } | d. } t t |
j |
j |
j |
j! Vd }
|
r~|
Vd }
| Vq* | d k rd } q* | d k r| d 7} n | d k r| d 8} |
r|
Vd }
t t" | | | | Vqt t | | | | f | | d f | V| d 7} qWq W|
rC |
Vd }
x; | d d D]) } t t d | d f | d f d VqT Wt t# d | d f | d f d Vd S)/Nr
0123456789r Fz utf-8-sigzutf-8r r zEOF in multi-line stringr z\
r z\
r z#
#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statementr! z...T
asyncawaitdef\z([{z)]})r r )r r )r r )r r r r r r r r )r r r )$r r
r r r r r r r
ERRORTOKENtabsizerstripr r r r IndentationErrorr rF PseudoTokenspanr r
triple_quotedendpats
single_quoted isidentifierr r r r( r1 r r r0 r ) r r lnumparenlev continuednumcharscontstrneedcontcontliner stashed async_defasync_def_indent async_def_nlr posmaxstrstartendprogendmatchr column
comment_tokennl_pos pseudomatchr sposeposr initialr r r- r- r. r sh
2
$#'
- ' $
'r c C s
t | d S)N)r )r r- r- r. generate_tokens s r c
s d d l } d d d d f d d } | j d d } | j d d d
d d d
d d | j d d d d d d d d | j } y | j r | j } t | d } t t | j } Wd QRXn d } t t
j j d } x[ | D]S } | j } | j
r| j } d | j | j } t d | t | | j f q WWnbt k
r}
z? |
j d d d \ } } | |
j d | | | f WYd d }
~
Xnt k
r}
z5 |
j d \ } } | |
j d | | | f WYd d }
~
Xn t k
r8}
z | |
| WYd d }
~
Xn t k
rf}
z | |
WYd d }
~
XnQ t k
rt d Yn6 t k
r}
z d |
WYd d }
~
Xn Xd S) Nr c S s t | d t j d S)Nfile)printsysstderr)messager- r- r. perror s zmain.<locals>.perrorc sg | r+ | f | | f } d | n+ | rH d | | f n d | t j d d S)Nz%s:%d:%d: error: %sz
%s: error: %sz error: %sr )r exit)r r locationargs)r r- r. error s zmain.<locals>.errorprogzpython -m tokenizedestr nargsr< metavarz filename.pyhelpz'the file to tokenize; defaults to stdinz-ez--exactexactaction
store_truez(display token names using the exact typer z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted
zunexpected error: %s)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr( r* r3 r r r r) r1 r r# r r OSErrorKeyboardInterrupt Exception)
r- r$ parserr# r fr r
token_type token_rangeerrr r r- )r r. main sN
//
r; __main__) r> r? rG rH rK rL rI rJ rM rN rO rP rS rT rQ rR rU rV rW rX r[ r\ rY rZ r] r^ r_ r` ra rb ) r@ rA ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r r r r r )}
__author__ __credits__builtinsr r codecsr r collectionsior r r rB r r rC ASCIIr r __all__N_TOKENSr r) r r
LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE
CIRCUMFLEX LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL
SLASHEQUAL PERCENTEQUAL
AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALATATEQUALr2
namedtupler r: r; r=
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumber StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtrasr rF r r r r r r5 r r r r r r
r r r r; r4 r- r- r- r. <module> sB
"
_ ] <