],U T @ s d Z d Z d d l Z d d l Z d d l m Z m Z d d l Td d l m Z d d e
e D d
d d g Z [ y e Wn e
k
r e Z Yn Xd
d Z d d Z d d Z d Z d Z e e d e e e Z d Z d Z d Z d Z d Z e e e e e Z d Z e d d e e Z d e Z e e e Z e d e d Z e e e e Z d! Z! d" Z" d# Z# d$ Z$ e d% d& Z% e d' d( Z& e d) d* d+ d, d- d. d/ d0 d1 Z' d2 Z( e d3 d4 Z) e e' e( e) Z* e e e* e& e Z+ e e+ Z, e d5 e d6 d d7 e d8 d Z- e d e e% Z. e e e. e e* e- e Z/ e0 e1 e j2 e, e/ e# e$ f \ Z3 Z4 Z5 Z6 d6 e j2 e! d8 e j2 e" d9 e5 d: e6 d; e5 d< e6 d= e5 d> e6 d? e5 d@ e6 dA e5 dB e6 dC e5 dD e6 dE e5 dF e6 dG e5 dH e6 dI e5 dJ e6 dK e5 dL e6 dM e5 dN e6 dO e5 dP e6 dQ e5 dR e6 dS e5 dT e6 dU e5 dV e6 dW d dX d dY d dZ d d[ d d\ d i& Z7 i Z8 x d D] Z9 e9 e8 e9 <qWi Z: x d D] Z9 e9 e: e9 <qWdy Z; Gdz d{ d{ e< Z= Gd| d} d} e< Z> d~ d Z? e? d d
Z@ d d ZA Gd d d ZB e j2 d e jC ZD e j2 d e jC ZE d d ZF d d ZG d d ZH d d ZI eJ d k rd d lK ZK eL eK jM d k re@ eN eK jM d jO n e@ eK jP jO d S)zKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro N)BOM_UTF8lookup)* )tokenc C s& g | ] } | d d k r | q S)r _ ).0xr r -/opt/alt/python35/lib64/python3.5/tokenize.py
<listcomp>% s r tokenizegenerate_tokens
untokenizec G s d d j | d S)N(|))join)choicesr r r group0 s r c G s t | d S)Nr )r )r r r r any1 s r c G s t | d S)N?)r )r r r r maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z
0[bB][01]*z0[xX][\da-fA-F]*[lL]?z0[oO]?[0-7]*[lL]?z
[1-9]\d*[lL]?z [eE][-+]?\d+z\d+\.\d*z\.\d+z\d+z\d+[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z[ubUB]?[rR]?'''z[ubUB]?[rR]?"""z&[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'z&[uU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz[:;.,`@]z'[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'z'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"'''"""r'''r"""u'''u"""b'''b"""ur'''ur"""br'''br"""R'''R"""U'''U"""B'''B"""uR'''uR"""Ur'''Ur"""UR'''UR"""bR'''bR"""Br'''Br"""BR'''BR"""rRuUbBr'r"R'R"u'u"U'U"b'b"B'B"ur'ur"Ur'Ur"uR'uR"UR'UR"br'br"Br'Br"bR'bR"BR'BR" c @ s e Z d Z d S)
TokenErrorN)__name__
__module__ __qualname__r r r r r] s r] c @ s e Z d Z d S)StopTokenizingN)r^ r_ r` r r r r ra s ra c C sF | \ } } | \ } } t d | | | | t | t | f d S)Nz%d,%d-%d,%d: %s %s)printtok_namerepr) typer Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr r r
printtoken s rg c C s* y t | | Wn t k
r% Yn Xd S)N)
tokenize_loopra )readline
tokeneaterr r r r
s
c C s% x t | D] } | | q
Wd S)N)r )ri rj Z
token_infor r r rh s rh c @ s@ e Z d Z d d Z d d Z d d Z d d Z d S)
Untokenizerc C s g | _ d | _ d | _ d S)Nr r )tokensprev_rowprev_col)selfr r r __init__ s zUntokenizer.__init__c C s7 | \ } } | | j } | r3 | j j d | d S)N )rn rl append)ro startrowcol
col_offsetr r r add_whitespace s
zUntokenizer.add_whitespacec C s x | D] } t | d k r0 | j | | P| \ } } } } } | j | | j j | | \ | _ | _ | t t f k r | j d 7_ d | _ q Wd j | j S)N r r )
lencompatrw rl rr rm rn NEWLINENLr )ro iterablettok_typer rs endrf r r r r s
zUntokenizer.untokenizec C s% d } g } | j j } | \ } } | t t f k r@ | d 7} | t t f k rX d } x | D] } | d d \ } } | t t t t f k r | d 7} | t k r | j | q_ nW | t k r | j
q_ n; | t t f k r d } n | r| r| | d d } | | q_ Wd S)NFrq Trx r ) rl rr NAMENUMBERr| r} ASYNCAWAITINDENTDEDENTpop) ro r r~ startlineindents toks_appendtoknumtokvaltokr r r r{ s0
zUntokenizer.compatN)r^ r_ r` rp rw r r{ r r r r rk s rk z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sd | d d j j d d } | d k s= | j d rA d S| d k s\ | j d
r` d S| S)N r -zutf-8zutf-8-latin-1
iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)r r r )r r r )lowerreplace
startswith)orig_encencr r r _get_normal_name s " r c s d d } d } f d d } f d d } | } | j t rj d | d d } d } | sz | g f S| | } | r | | g f St j | s | | g f S| } | s | | g f S| | } | r | | | g f S| | | g f S)
NFzutf-8c s+ y SWn t k
r& t SYn Xd S)N)
StopIterationbytesr )ri r r read_or_stop s
z%detect_encoding.<locals>.read_or_stopc s y | j d } Wn t k
r+ d SYn Xt j | } | sE d St | j d } y t | } Wn" t k
r t d | Yn X r | j d k r t d | d 7} | S)Nasciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError cookie_rematchr r r LookupError SyntaxErrorname)rf line_stringr encodingcodec) bom_foundr r find_cookie s"
z$detect_encoding.<locals>.find_cookieT z utf-8-sig)r r blank_rer )ri r defaultr r firstsecondr )r ri r detect_encoding s0
r c C s t } | j | S)N)rk r )r~ utr r r r F s c ! c sY d } } } t j d d } } d \ } } d } d g } d }
d } d } d }
xy
| } Wn t k
r d } Yn X| d } d t | } } | r| s t d | | j | } | r!| j d } } t | | d | | | | f | | f Vd \ } } d } qv| r| d! d d k r| d" d d k rt | | | | t | f | f Vd } d } qX qv| | } | | } qX n| d k rU| rU| sPd } xv | | k r;| | d k r| d } n? | | d
k r| t d t } n | | d k r-d } n P| d } qW| | k rIP|
rZ|
Vd }
| | d k rG| | d k r | | d j
d } | t | } t | | | f | | t | f | f Vt | | d | | f | t | f | f VqX t t f | | d k | | d | | f | t | f | f VqX | | d# k r| j
| t | d | | d f | | f | f Vx | | d$ k r#| | k rt d d | | | f | d d% } | r | | d& k r d } d }
d } t d | | f | | f | f VqW| rv|
rv| | d' k rvd } d }
d } n! | spt d | d f d } xl| | k rt j | | } | r| j d \ } } | | f | | f | } } } | | | | | } } | | k s| d k r(| d k r(t | | | | f Vq| d k rt } | d k rOt } n | r[d }
|
rl|
Vd }
| | | | | f Vq| d k r|
r|
Vd }
t | | | | f Vq| t k rXt | } | j | | } | r2| j d } | | | } |
r|
Vd }
t | | | | f | f Vq| | f } | | d } | } Pq| t k s| d d t k s| d d
t k r| d( d k r| | f } t | pt | d pt | d } | | d d } } | } Pq|
r|
Vd }
t | | | | f Vq| | k r| d) k rb| rb| d k rKt n t | | | | f Vqyt | | | | f } | d k r|
r| }
qy| d k r|
r|
d t k r|
d d k rd } | d* } t |
d |
d |
d
|
d f Vd }
|
r|
Vd }
| Vq| d k rX|
r5|
Vd }
t | | | | f | f Vd } q| d k rq| d } n | d k r| d } |
r|
Vd }
t | | | | f Vqyt | | | | f | | d f | f V| d } qyWqX W|
r|
Vd }
x8 | d d D]& } t d | d f | d f d f Vq Wt d | d f | d f d f Vd S)+Nr r
0123456789ry Fr zEOF in multi-line stringrx z\
r z\
rq z#
#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement.T
asyncawaitdef \z([{z)]})ry r )ry r r r r r r r )r r r ) string
ascii_lettersr rz r] r r STRING
ERRORTOKENtabsizerstripCOMMENTr} rr r IndentationErrorr
pseudoprogspanr r|
triple_quotedendprogs
single_quotedr r r OP ENDMARKER)!ri lnumparenlev continuedZ namecharsnumcharscontstrneedcontcontliner stashed async_defasync_def_indent async_def_nlrf posmaxstrstartendprogendmatchr column
comment_tokennl_pos pseudomatchrs sposeposr initialnewliner indentr r r r [ sp
2
$#$
* $
$__main__) r r r r r( r) r r! r* r+ r" r# r, r- r$ r% r0 r1 r. r/ r2 r3 r& r' r6 r7 r4 r5 r8 r9 ) r r r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ )Q
__author__ __credits__r recodecsr r Zlib2to3.pgen2.tokenry r dir__all__r NameErrorstrr r r
WhitespaceCommentIgnoreName Binnumber Hexnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtras PseudoTokenlistmapcompileZ tokenprogr Z single3progZ double3progr r r r r Exceptionr] ra rg r
rh rk ASCIIr r r r r r r^ sysrz argvopenri stdinr r r r <module> s
*
8 I