U
V
@ s" d Z d ZddlZddlZddlZddlZddlZddlZzejej fZ
W n e k
rd e e
fZ
Y nX edZG dd deZG dd deZG dd d eZG d
d d eZG d d
d
Zdd Zdd Zdd Zdd Zdd Zdd Z G dd deZ d$d d Z d%d d!Z d"d# Z e Z!dS )&z3.8 Nz^[a-zA-Z0-9_]+$c @ s e Zd Zdd ZdS )LexErrorc C s |f| _ || _d S N)argstext)selfmessages r /lex.py__init__: s zLexError.__init__N)__name__
__module__ __qualname__r r r r r
r 9 s r c @ s e Zd Zdd Zdd ZdS )LexTokenc C s d| j | j| j| jf S )NzLexToken(%s,%r,%d,%d))typevaluelinenolexposr r r r
__str__A s zLexToken.__str__c C s t | S r )strr r r r
__repr__D s zLexToken.__repr__N)r r
r r r r r r r
r @ s r c @ s4 e Zd Zdd Zdd Zdd Zdd ZeZeZd S )
PlyLoggerc C s
|| _ d S r f)r r r r r
r L s zPlyLogger.__init__c O s | j || d d S )N
r writer msgr kwargsr r r
criticalO s zPlyLogger.criticalc O s | j d|| d d S )Nz WARNING: r r r r r r
warningR s zPlyLogger.warningc O s | j d|| d d S )NzERROR: r r r r r r
errorU s zPlyLogger.errorN) r r
r r r! r" r# infodebugr r r r
r K s r c @ s e Zd Zdd Zdd ZdS )
NullLoggerc C s | S r r )r namer r r
__getattribute__^ s zNullLogger.__getattribute__c O s | S r r )r r r r r r
__call__a s zNullLogger.__call__N)r r
r r( r) r r r r
r& ] s r& c @ s| e Zd Zdd Zd ddZd ddZd d
Zd d Zd
d Zdd Z dd Z
dd Z dd Z dd Z
dd Zdd ZeZdS ) Lexerc C s d | _ d | _i | _i | _i | _d| _g | _d | _i | _i | _ i | _
d| _ d | _ d| _
d| _d | _d | _d | _d| _d| _d | _d| _d| _d S )NINITIALr F)lexre lexretext
lexstaterelexstateretextlexstaterenameslexstate
lexstatestack lexstateinfolexstateignorelexstateerrorf lexstateeoff
lexreflagslexdatar lexlen lexerrorflexeoff lextokens lexignore lexliterals lexmoduler lexoptimizer r r r
r t s. zLexer.__init__Nc C s t | }|ri }| j D ]t\}}g }|D ]L\}}g } |D ]:}
|
rL|
d sX| |
q<| t||
d j|
d f q<q,||| f |||< q ||_i |_| j D ]\}} t|| j|j|< q||_|S Nr r- )copyr0 itemsappendgetattrr r7 rA ) r objectcZnewtabkeyZritemZnewreZcreZfindexZ newfindexr efr r r
clone s(
"
z Lexer.cloner, c C s t |tjrtd|dd }tj||d }t|d}| d|t
f | dt t | dt | j
| d t | j | d
t | j | d t | j i }| j D ]P\}}g } t|| j| | j| D ]"\\}
} } }
| | t| |
f q| ||< q| d t | | d
t | j i }| j D ] \}}|r^|jnd ||< qJ| dt | i }| j D ] \}}|r|jnd ||< q| dt | W 5 Q R X d S )Nz&Won't overwrite existing lextab module.z.pywzJ# %s.py. This file automatically created by PLY (version %s). Don't edit!
z_tabversion = %s
z_lextokens = %s
z_lexreflags = %s
z_lexliterals = %s
z_lexstateinfo = %s
z_lexstatere = %s
z_lexstateignore = %s
z_lexstateerrorf = %s
z_lexstateeoff = %s
)
isinstancetypes
ModuleTypeIOErrorsplitospathjoinopenr __version__repr__tabversion__r> r9 r@ r5 r0 rE zipr1 r2 rF _funcs_to_namesr6 r7 r r8 )r lextab outputdirZ
basetabmodulefilenameZtfZtabre statenamelretitempatfuncZretextrenamesZtaberrrK Ztabeofr r r
writetab s6 &
zLexer.writetabc C sH t |tjr|}ntd| tj| }t|ddtkr@td|j | _
|j | _ |j
| _| j
t| jB | _|j| _|j| _i | _i | _|j D ]V\}}g }g }|D ],\}} |t||j tj B t | |f q|| j|< || j|< qi | _ |j D ]\}}
||
| j |< qi | _ |j! D ]\}}
||
| j |< q | "d d S )N import %sZ _tabversionz0.0zInconsistent PLY versionr+ )#rP rQ rR execsysmodulesrG r[ ImportErrorZ
_lextokensr> Z _lexreflagsr9 Z _lexliteralsr@ set
lextokens_allZ
_lexstateinfor5 Z_lexstateignorer6 r0 r1 Z _lexstatererE rF recompileVERBOSE_names_to_funcsr7 Z_lexstateerrorfr8 Z
_lexstateeoffbegin) r Ztabfilefdictr^ ra rb rc Ztxtitemrd Z func_namerK r r r
readtab s8
&
z
Lexer.readtabc C s8 |d d }t |ts td|| _d| _t|| _d S )Nr- zExpected a stringr )rP StringTypes
ValueErrorr: r lenr; )r r rI r r r
input s
z Lexer.inputc C sd || j krtd| j | | _| j| | _| j|d| _| j|d | _ | j
|d | _ || _ d S )NzUndefined stater, )
r0 rw r. r1 r/ r6 getr? r7 r< r8 r= r3 r stater r r
rs s
z Lexer.beginc C s | j | j | | d S r )r4 rF r3 rs r{ r r r
push_state s zLexer.push_statec C s | | j d S r )rs r4 popr r r r
pop_state s zLexer.pop_statec C s | j S r )r3 r r r r
current_state! s zLexer.current_statec C s | j |7 _ d S r )r )r nr r r
skip' s z
Lexer.skipc C s~ | j }| j}| j}| j}||k r|| |kr8|d7 }q| jD ]\}}|||}|sXq>t }| |_| j |_ ||_ |j
} || \}
|_ |
s|j r| | _ | S | } q| }| |_
|| _|| _ |
|} | s| j }| j} q| js(| j | jkr(td|
jj|
jj|
j| j f ||d | S || | jkrtt }|| |_| j |_ |j|_ ||_ |d | _ |S | jrt }| j|d |_| j |_ d|_ | |_
||_ || _ | |} || j krtd|| ||d | j }| sq| S || _ td|| |f ||d q| jr\t }d|_ d|_| j |_ ||_ | |_
|| _ | |} | S |d | _ | jd krztdd S ) Nr- z4%s:%d: Rule '%s' returned an unknown token type '%s'r# z&Scanning error. Illegal character '%s'z"Illegal character '%s' at index %deofr, z"No input string given with input())r r; r? r: r. matchr groupr r lastindexr endlexerZlexmatchrB rn r __code__ co_filenameco_firstlinenor r@ r< r= RuntimeError) r r r; r? r: r. lexindexfuncmtokire Znewtokr r r
token1 s
z Lexer.tokenc C s | S r r r r r r
__iter__ s zLexer.__iter__c C s | }|d krt|S r )r
StopIteration)r tr r r
next s z
Lexer.next)N)r, )r r
r r rL rg ru ry rs r} r r r r r r __next__r r r r
r* s s
%(
nr* c C s t | d| jS )Nregex)rG __doc__)re r r r
_get_regex s r c C s0 t | }|j }|j|jkr,||j |S r )rj _getframe f_globalsrD f_localsupdate)Zlevelsr ldictr r r
get_caller_module_dict s
r c C sF g }t | |D ]2\}}|r6|d r6|||d f q|| q|S rC )r\ rF )Zfunclistnamelistresultr r' r r r
r] s r] c C sD g }| D ]6}|r4|d r4| ||d |d f q| | q|S rC )rF )r rt r r r r r
rr s rr c C sl | sg S d | }zt|tj|B }d gt|j d }|d d }|j D ]z\}} ||d }
t |
t
j t
j fkr|
|| f|| < ||| < qT|
d k rT||| < |
ddkrd|| < qTd || f|| < qT||fg|g|gfW S tk
rf tt| d } | dkrd} t| d | |||\} }
}t| | d |||\}}}| | |
| || f Y S X d S )N|r- ignore_r )NN )rW ro rp rq max
groupindexvaluesrE rz r rQ FunctionType
MethodTypefind Exceptionintrx _form_master_re)Zrelistreflagsr toknamesr r. r Z
lexindexnamesr r Zhandler Zllistrb ZlnamesZrlistZrreZrnamesr r r
r s2
r c C s d}| d}t|dd dD ] \}}||kr |dkr q>q |dkrXt|d| }nd}d|krlt|}d||d }||fS )Nr- _ANY)r+ )rT enumeratetuplerW )r namesZnonstatepartsr partstatesZ tokennamer r r
_statetoken s
r c @ sf e Zd ZdddZdd Zdd Zd d
Zd d Zd
d Zdd Z dd Z
dd Z dd Z dd Z
dS ) LexerReflectNr c C sL || _ d | _g | _|| _ddi| _t | _d| _|d krBtt j
n|| _ d S )Nr+ inclusiveF) r Z
error_functokensr stateinform rk r# r rj stderrlog)r r r r r r r
r / s
zLexerReflect.__init__c C s$ | | | | d S r )
get_tokens get_literals
get_states get_rulesr r r r
get_all: s zLexerReflect.get_allc C s | | | | jS r )validate_tokensvalidate_literalsvalidate_rulesr# r r r r
validate_allA s zLexerReflect.validate_allc C sp | j dd }|s(| jd d| _d S t|ttfsL| jd d| _d S |sf| jd d| _d S || _d S )Nr zNo token list is definedTz tokens must be a list or tupleztokens is empty)r rz r r# rP listr r )r r r r r
r H s zLexerReflect.get_tokensc C sP i }| j D ]@}t|s,| jd| d| _||krB| jd| d||< q
d S )NzBad token name '%s'TzToken '%s' multiply definedr- )r _is_identifierr r r# r" )r Z terminalsr r r r
r \ s
z LexerReflect.validate_tokensc C s | j dd| _| js d| _d S )Nliteralsr, )r rz r r r r r
r g s zLexerReflect.get_literalsc C sj z>| j D ]2}t|tr"t|dkr| jdt| d| _qW n& tk
rd | jd d| _Y nX d S )Nr- z.Invalid literal %s. Must be a single characterTzIInvalid literals specification. literals must be a sequence of characters)r rP rv rx r r# rZ TypeError)r rI r r r
r m s
z LexerReflect.validate_literalsc C s | j dd | _| jrt| jttfs:| jd d| _n| jD ]}t|trZt|dkrt| jdt | d| _q@|\}}t|t
s| jdt | d| _q@|dks|dks| jd | d| _q@|| j kr| jd
| d| _q@|| j |< q@d S ) Nr z)states must be defined as a tuple or listTr zMInvalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')z State name %s must be a stringr exclusivez:State type for state %s must be 'inclusive' or 'exclusive'zState '%s' already defined) r rz r rP r r r r# rx rZ rv r )r r r' Z statetyper r r
r x s0
zLexerReflect.get_statesc C s, dd | j D }i | _i | _i | _i | _i | _i | _| jD ]}g | j|< g | j|< q:t|dkrv| j
d d| _
d S |D ]d}| j | }t || j\}}|| j|< t |drD|dkr|D ]}|| j|< qnt|dkr|D ]}|| j|< qnV|d kr"|j
j}|j
j}| j
d
|||j d| _
n |D ]}| j| ||f q&qzt|tr|d kr|D ]}|| j|< q^d |kr| j d | n@|dkr| j
d
| d| _
n |D ]}| j| ||f qqz| j
d| d| _
qz| j D ]}|jdd d q| j D ]}|jdd dd qd S )Nc S s g | ]}|d d dkr|qS )Nr Zt_r ).0r r r r
<listcomp> s z*LexerReflect.get_rules.<locals>.<listcomp>r z+No rules of the form t_rulename are definedTr) r# r ignorez,%s:%d: Rule '%s' must be defined as a string\z#%s contains a literal backslash '\''Rule '%s' must be defined as a functionz&%s not defined as a function or stringc S s | d j jS Nr- )r r xr r r
<lambda> z(LexerReflect.get_rules.<locals>.<lambda>)rJ c S s t | d S r )rx r r r r
r r )rJ reverse)r r funcsymstrsymr errorfeoffr rx r r# r hasattrr r r r rF rP rv r" r sort) r Ztsymbolsr r r r toknamelinefiler r r
r sb
zLexerReflect.get_rulesc C s | j D ]b}| j| D ]r\}}|jj}|jj}t|}| j| | j | }t
|t j rbd}nd}|jj
} | |kr| jd|||j d| _q| |k r| jd|||j d| _qt|s| jd|||j d| _qzJtd|t|f tj| jB }
|
dr"| jd |||j d| _W q tjk
r } zB| jd
|||j| d t|krr| jd |||j d| _W 5 d } ~ X Y qX q| j| D ]\} }
| j | }|d
kr| jd| d| _q|| jkr|ddk r| jd| | d| _qz@td| |
f tj| jB }
|
dr@| jd| d| _W nT tjk
r } z2| jd| | d |
kr| jd| d| _W 5 d } ~ X Y nX q| j| s| j| s| jd| d| _| j|d }|r|}|jj}|jj}t|}| j| t
|t j rd}nd}|jj
} | |krH| jd|||j d| _| |k r| jd|||j d| _q| jD ]}| | qrd S )Nr r- z'%s:%d: Rule '%s' has too many argumentsTz%%s:%d: Rule '%s' requires an argumentz2%s:%d: No regular expression defined for rule '%s'
(?P<%s>%s)r, z<%s:%d: Regular expression for rule '%s' matches empty stringz3%s:%d: Invalid regular expression for rule '%s'. %s#z6%s:%d. Make sure '#' in rule '%s' is escaped with '\#'r# r r r z-Rule '%s' defined for an unspecified token %sz5Regular expression for rule '%s' matches empty stringz,Invalid regular expression for rule '%s'. %sz/Make sure '#' in rule '%s' is escaped with '\#'z No rules defined for state '%s') r r r r r inspectZ getmodulerk addr rP rQ r co_argcountr r# r r ro rp rq r r r r r r rz validate_module)r r| fnamer r r moduler ZreqargsnargsrI er' rZefuncr r r
r s
zLexerReflect.validate_rulesc C s t |\}}td}td}i }|d7 }|D ]n}||}|sN||}|r|d} || }
|
st||| < n$t |} | j d| || |
d| _ |d7 }q2d S )Nz \s*def\s+(t_[a-zA-Z_0-9]*)\(z\s*(t_[a-zA-Z_0-9]*)\s*=r- z7%s:%d: Rule %s redefined. Previously defined on line %dT)
r Zgetsourcelinesro rp r r rz Z
getsourcefiler r# ) r r linesZlinenZfreZsreZ counthashr r r' prevr` r r r
r @ s$
z LexerReflect.validate_module)Nr )r r
r r r r r r r r r r r r r r r r
r . s
Bgr Fr^ c
%
sr |d kr d}d }
ddi} t } || _| d kr6ttj} |rL|d krLttj}|rT| r fddt D }
t|
}
d|
krtj|
d j|
d< nt d}
|
d }|rt |t rd
|kr|d
| }t
|
| |d }| |s| rtd |r8|r8z"| ||
| ja| ja| a| W S tk
r6 Y nX |rh|d
|j |d|j |d|j t | _|jD ]}| j | qvt |jt t frt |jd |j| _!n|j| _!| jt| j!B | _"|j} i }| D ]}g }|j#| D ]J\}}|j$j%}|j$j&}|'d|t(|f |r|d|t(|| q|j)| D ]2\}}|'d||f |rH|d||| qH|||< q|r|d |D ]h}t*|| ||
|j+\}}} || j,|< || j-|< | | j.|< |rt/|D ]\} } |d|| | qq| 0 D ]`\}} |dkr| dkr| j,| 1| j,d | j-| 1| j-d | j.| 1| j.d q| | _2| j,d | _3| j-d | _4|| _5|j6| _7| j7
dd| _8|j9| _:|j9
dd | _;| j;s| <d |j=| _>|j=
dd | _?| 0 D ]\} } | dkrB| |j9kr | <d| | |j6kr| j8r| <d| nJ| dkr| |j9krl|j9
dd |j9| < | |j6kr|j6
dd|j6| < q| ja| ja| a|rn|rn|d kr&t |t@jAr|j}!nNd
|kr|
d }!n:|Bd
}"d
|"d d }#tCd|# tDtj|# dd}!tEjFG|!}z| H|| W n6 tIk
rl }$ z| <d ||$f W 5 d }$~$X Y nX | S ) Nr^ r+ r c s g | ]}|t |fqS r )rG )r kr r r
r w s zlex.<locals>.<listcomp>__file__r
r __package__rM )r r zCan't build lexerzlex: tokens = %rzlex: literals = %rzlex: states = %rr r z(lex: Adding rule %s -> '%s' (state '%s')z#lex: ==== MASTER REGEXS FOLLOW ====z"lex: state '%s' : regex[%d] = '%s'r, zNo t_error rule is definedr z1No error rule is defined for exclusive state '%s'z2No ignore rule is defined for exclusive state '%s'rN rh z#Couldn't write lextab module %r. %s)Jr* rB r rj r dirdictrk r r rz rP r r r r SyntaxErrorru r ry r rl r$ r r r rm r> r r r r rW r@ rn r r r r rF r r r r r0 r1 r2 r rE extendr5 r. r/ r9 r r6 r? r r7 r< r" r r8 r= rQ rR rT ri rG rU rV dirnamerg rS )%r rH r% optimizer^ r Znowarnr_ ZdebuglogZerrorlogr r ZlexobjZ_itemsZpkgZlinfor Zregexsr| Z
regex_listr r r r r' r r. Zre_textZre_namesr r styper Zsrcfiler Zpkgnamer r r r
lex\ s
$r c C s |sVz&t jd }t|}| }| W n* tk
rT t jd t j }Y nX | rb| j }nt }|| | rz| j
}nt
}| }|sqt jd|j |j |j
|jf q~d S )Nr- z/Reading from standard input (type EOF to end):
z(%s,%r,%d,%d)
)rj argvrX readclose
IndexErrorstdoutr stdinry r r r r r )r datar` r Z_inputZ_tokenr r r r
runmain s(
r c s fdd}|S )Nc s t drt | _n | _| S )Nr) )r r r r r r r
set_regex? s
zTOKEN.<locals>.set_regexr )r r r r r
TOKEN> s r )
NNFFr^ r FNNN)NN)"rY r[ ro rj rQ rD rU r Z
StringTypeZ UnicodeTyperv AttributeErrorr bytesrp r r r rH r r r& r* r r r] rr r r r r r r Tokenr r r r
<module>" sN
F
( 0
A
"