3
aj$+ @ s d dl mZmZmZ d dlmZ d dlmZ ddl m
Z
ddl m Z ddl m Z m
Z
ddl mZmZmZ dd l mZmZ dd
l mZ dd lmZ dd lmZ ee ZG d
d deZdS ) )absolute_importdivisionunicode_literals)unichr)deque )spaceCharacters)entities) asciiLettersasciiUpper2Lower)digits hexDigitsEOF)
tokenTypes
tagTokenTypes)replacementCharacters)HTMLInputStream)Triec sd e Zd ZdZd fdd Zdd Zdd Zdd
d Zd d
Zdd Z dd Z
dd Z dd Z dd Z
dd Zdd Zd d Zd d Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Z d6d7 Z d8d9 Z d:d; Z d<d= Z d>d? Z!d@dA Z"dBdC Z#dDdE Z$dFdG Z%dHdI Z&dJdK Z'dLdM Z(dNdO Z)dPdQ Z*dRdS Z+dTdU Z,dVdW Z-dXdY Z.dZd[ Z/d\d] Z0d^d_ Z1d`da Z2dbdc Z3ddde Z4dfdg Z5dhdi Z6djdk Z7dldm Z8dndo Z9dpdq Z:drds Z;dtdu Z<dvdw Z=dxdy Z>dzd{ Z?d|d} Z@d~d ZAdd ZBdd ZCdd ZDdd ZEdd ZFdd ZGdd ZHdd ZIdd ZJdd ZKdd ZL ZMS )
HTMLTokenizera This class takes care of tokenizing HTML.
* self.currentToken
Holds the token that is currently being processed.
* self.state
Holds a reference to the method to be invoked... XXX
* self.stream
Points to HTMLInputStream object.
Nc sF t |f|| _|| _d| _g | _| j| _d| _d | _t t
| j d S )NF) r streamparserZ
escapeFlagZ
lastFourChars dataStatestateescape currentTokensuperr __init__)selfr r kwargs) __class__ /usr/lib/python3.6/_tokenizer.pyr " s zHTMLTokenizer.__init__c c s\ t g | _xL| j rVx&| jjr:td | jjjddV qW x| jrR| jj V q>W q W dS )z This is where the magic happens.
We do our usually processing through the states and when we have a token
to return we yield the token which pauses processing until the next token
is requested.
ParseErrorr )typedataN)r
tokenQueuer r errorsr poppopleft)r r r r! __iter__1 s
zHTMLTokenizer.__iter__c % C s( t }d}|rt}d}g }| jj }x(||krJ|tk rJ|j| | jj }q$W tdj||}|tkrt| }| j jt
d dd|id nld| kod kn s|d
krd }| j jt
d dd|id n(d | kod
kn sd| kodkn sd| kodkn sd| ko4dkn s|t ddddddddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d
g#kr| j jt
d dd|id y t |}W n> t
k
r |d6 }t d|d? B t d7|d8@ B }Y nX |d9kr$| j jt
d d:d; | jj| |S )<zThis function returns either U+FFFD or the character based on the
decimal or hexadecimal representation. It also discards ";" if present.
If not present self.tokenQueue.append({"type": tokenTypes["ParseError"]}) is invoked.
r" z$illegal-codepoint-for-numeric-entity charAsInt)r# r$ datavarsi i i u �r i i i i i i i i i i i i i i i i i i i i i i i
i
i i i i i
i
i i i i i i i i ;z numeric-entity-without-semicolon)r# r$ )r r
r charr appendintjoinr r% r frozensetchr
ValueErrorunget) r ZisHexZallowedradix charStackcr- r6 vr r r! consumeNumberEntityA s`
&
z!HTMLTokenizer.consumeNumberEntityFc
C s d}| j j g}|d tksB|d tddfksB|d k rV||d krV| j j|d n"|d dkrd}|j| j j |d dkrd }|j| j j |r|d tks| r|d tkr| j j|d | j|}n4| j jt
d
d d | j j|j dd
j | }njx8|d tk rFt
jd
j |s2P |j| j j qW y$t
jd
j |d d }t|}W n tk
r d }Y nX |d k rD|d dkr| j jt
d
dd |d dkr|r|| tks|| tks|| dkr| j j|j dd
j | }n.t| }| j j|j |d
j ||d 7 }n4| j jt
d
dd | j j|j dd
j | }|r| jd d d |7 < n*|tkrd}nd}| j jt
| |d d S ) N&r <#Fr xXTr" zexpected-numeric-entity)r# r$ r, r5 z named-entity-without-semicolon=zexpected-named-entityr$ SpaceCharacters
Characters)rF rG rK rK rK rK rK rK rK rK )r r6 r r r= r7 r
r rB r% r r' r9 entitiesTrieZhas_keys_with_prefixZlongest_prefixlenKeyErrorr
r r ) r allowedChar
fromAttributeoutputr? hexZ
entityNameZ entityLengthZ tokenTyper r r!
consumeEntity sf
zHTMLTokenizer.consumeEntityc C s | j |dd dS )zIThis method replaces the need for "entityInAttributeValueState".
T)rO rP N)rS )r rO r r r! processEntityInAttribute s z&HTMLTokenizer.processEntityInAttributec C s | j }|d tkrp|d jt|d< |d td krp|d rR| jjtd dd |d rp| jjtd d d | jj| | j| _d
S ) zThis method is a generic handler for emitting the tags. It also sets
the state to "data" because that's what's needed after a token has been
emitted.
r# nameEndTagr$ r" zattributes-in-end-tag)r# r$ selfClosingz self-closing-flag-on-end-tagN) r r translater r r% r7 r r )r tokenr r r! emitCurrentToken s
z HTMLTokenizer.emitCurrentTokenc C s | j j }|dkr | j| _n|dkr.| j| _n|dkrd| jjtd dd | jjtd dd n`|tkrpdS |t kr| jjtd || j j
t d
d n&| j j
d }| jjtd || d d
S ) NrC rD r" zinvalid-codepoint)r# r$ rJ FrI T)rC rD r[ ) r r6 entityDataStater tagOpenStater% r7 r r r
charsUntil)r r$ charsr r r! r s&
zHTMLTokenizer.dataStatec C s | j | j| _dS )NT)rS r r )r r r r! r\ s z HTMLTokenizer.entityDataStatec C s | j j }|dkr | j| _n|dkr.| j| _n|tkr:dS |dkrp| jjtd dd | jjtd d d nT|t kr| jjtd
|| j j
t d d n&| j j
d }| jjtd || d d S )
NrC rD Fr[ r" zinvalid-codepoint)r# r$ rJ u �rI T)rC rD r[ ) r r6 characterReferenceInRcdatar rcdataLessThanSignStater r% r7 r r r^ )r r$ r_ r r r! rcdataState s&
zHTMLTokenizer.rcdataStatec C s | j | j| _dS )NT)rS rb r )r r r r! r` 1 s z(HTMLTokenizer.characterReferenceInRcdatac C s | j j }|dkr | j| _nh|dkrR| jjtd dd | jjtd dd n2|tkr^dS | j jd
}| jjtd || d d S ) NrD r[ r" zinvalid-codepoint)r# r$ rJ u �FT)rD r[ ) r r6 rawtextLessThanSignStater r% r7 r r r^ )r r$ r_ r r r! rawtextState6 s
zHTMLTokenizer.rawtextStatec C s | j j }|dkr | j| _nh|dkrR| jjtd dd | jjtd dd n2|tkr^dS | j jd
}| jjtd || d d S ) NrD r[ r" zinvalid-codepoint)r# r$ rJ u �FT)rD r[ ) r r6 scriptDataLessThanSignStater r% r7 r r r^ )r r$ r_ r r r! scriptDataStateH s
z HTMLTokenizer.scriptDataStatec C sr | j j }|tkrdS |dkrL| jjtd dd | jjtd dd n"| jjtd || j jd d dS ) NFr[ r" zinvalid-codepoint)r# r$ rJ u �T)r r6 r r% r7 r r^ )r r$ r r r! plaintextStateZ s
z HTMLTokenizer.plaintextStatec C s | j j }|dkr | j| _n|dkr.| j| _n|tkrVtd |g ddd| _| j| _n|dkr| j j
td dd | j j
td
d d | j | _nt|d kr| j j
td d
d | j j | | j
| _n@| j j
td dd | j j
td
dd | j j | | j | _dS )N!/ZStartTagF)r# rU r$ rW ZselfClosingAcknowledged>r" z'expected-tag-name-but-got-right-bracket)r# r$ rJ z<>?z'expected-tag-name-but-got-question-markzexpected-tag-namerD T)r r6 markupDeclarationOpenStater closeTagOpenStater
r r tagNameStater% r7 r r= bogusCommentState)r r$ r r r! r] i s6
zHTMLTokenizer.tagOpenStatec C s | j j }|tkr0td |g dd| _| j| _n|dkrX| jjtd dd | j | _nn|t
kr| jjtd dd | jjtd d
d | j | _n0| jjtd d d |id
| j j | | j | _dS )NrV F)r# rU r$ rW rj r" z*expected-closing-tag-but-got-right-bracket)r# r$ z expected-closing-tag-but-got-eofrJ z</z!expected-closing-tag-but-got-charr$ )r# r$ r. T)
r r6 r
r r rn r r% r7 r r r= ro )r r$ r r r! rm s(
z HTMLTokenizer.closeTagOpenStatec C s | j j }|tkr | j| _n|dkr.| j n~|tkrV| jjt d dd | j
| _nV|dkrh| j | _nD|dkr| jjt d dd | j d d 7 < n| j d |7 < d
S ) Nrj r" zeof-in-tag-name)r# r$ ri r[ zinvalid-codepointrU u �T)
r r6 r beforeAttributeNameStater rZ r r% r7 r r selfClosingStartTagStater )r r$ r r r! rn s"
zHTMLTokenizer.tagNameStatec C sP | j j }|dkr"d| _| j| _n*| jjtd dd | j j| | j | _dS )Nri r, rJ rD )r# r$ T)
r r6 temporaryBufferrcdataEndTagOpenStater r% r7 r r= rb )r r$ r r r! ra s
z%HTMLTokenizer.rcdataLessThanSignStatec C sX | j j }|tkr*| j|7 _| j| _n*| jjtd dd | j j | | j
| _dS )NrJ z</)r# r$ T) r r6 r
rr rcdataEndTagNameStater r% r7 r r= rb )r r$ r r r! rs s
z#HTMLTokenizer.rcdataEndTagOpenStatec C s | j o | j d j | jj k}| jj }|tkrT|rTtd | jg dd| _ | j| _n|dkr|rtd | jg dd| _ | j | _n||dkr|rtd | jg dd| _ | j
| j | _nH|t kr| j|7 _n0| j
jtd d| j d | jj| | j| _d
S ) NrU rV F)r# rU r$ rW ri rj rJ z</)r# r$ T)r lowerrr r r6 r r rp r rq rZ r r
r% r7 r= rb )r appropriater$ r r r! rt s2
z#HTMLTokenizer.rcdataEndTagNameStatec C sP | j j }|dkr"d| _| j| _n*| jjtd dd | j j| | j | _dS )Nri r, rJ rD )r# r$ T)
r r6 rr rawtextEndTagOpenStater r% r7 r r= rd )r r$ r r r! rc s
z&HTMLTokenizer.rawtextLessThanSignStatec C sX | j j }|tkr*| j|7 _| j| _n*| jjtd dd | j j | | j
| _dS )NrJ z</)r# r$ T) r r6 r
rr rawtextEndTagNameStater r% r7 r r= rd )r r$ r r r! rw s
z$HTMLTokenizer.rawtextEndTagOpenStatec C s | j o | j d j | jj k}| jj }|tkrT|rTtd | jg dd| _ | j| _n|dkr|rtd | jg dd| _ | j | _n||dkr|rtd | jg dd| _ | j
| j | _nH|t kr| j|7 _n0| j
jtd d| j d | jj| | j| _d
S ) NrU rV F)r# rU r$ rW ri rj rJ z</)r# r$ T)r ru rr r r6 r r rp r rq rZ r r
r% r7 r= rd )r rv r$ r r r! rx s2
z$HTMLTokenizer.rawtextEndTagNameStatec C sx | j j }|dkr"d| _| j| _nR|dkrJ| jjtd dd | j| _n*| jjtd dd | j j | | j
| _dS ) Nri r, rh rJ z<!)r# r$ rD T) r r6 rr scriptDataEndTagOpenStater r% r7 r scriptDataEscapeStartStater= rf )r r$ r r r! re s
z)HTMLTokenizer.scriptDataLessThanSignStatec C sX | j j }|tkr*| j|7 _| j| _n*| jjtd dd | j j | | j
| _dS )NrJ z</)r# r$ T) r r6 r
rr scriptDataEndTagNameStater r% r7 r r= rf )r r$ r r r! ry , s
z'HTMLTokenizer.scriptDataEndTagOpenStatec C s | j o | j d j | jj k}| jj }|tkrT|rTtd | jg dd| _ | j| _n|dkr|rtd | jg dd| _ | j | _n||dkr|rtd | jg dd| _ | j
| j | _nH|t kr| j|7 _n0| j
jtd d| j d | jj| | j| _d
S ) NrU rV F)r# rU r$ rW ri rj rJ z</)r# r$ T)r ru rr r r6 r r rp r rq rZ r r
r% r7 r= rf )r rv r$ r r r! r{ 7 s2
z'HTMLTokenizer.scriptDataEndTagNameStatec C sJ | j j }|dkr2| jjtd dd | j| _n| j j| | j| _dS )N-rJ )r# r$ T) r r6 r% r7 r scriptDataEscapeStartDashStater r= rf )r r$ r r r! rz S s
z(HTMLTokenizer.scriptDataEscapeStartStatec C sJ | j j }|dkr2| jjtd dd | j| _n| j j| | j| _dS )Nr| rJ )r# r$ T) r r6 r% r7 r scriptDataEscapedDashDashStater r= rf )r r$ r r r! r} ] s
z,HTMLTokenizer.scriptDataEscapeStartDashStatec C s | j j }|dkr2| jjtd dd | j| _n|dkrD| j| _nn|dkrz| jjtd dd | jjtd dd n8|tkr| j | _n&| j j
d
}| jjtd || d d S ) Nr| rJ )r# r$ rD r[ r" zinvalid-codepointu �T)rD r| r[ ) r r6 r% r7 r scriptDataEscapedDashStater "scriptDataEscapedLessThanSignStater r r^ )r r$ r_ r r r! scriptDataEscapedStateg s"
z$HTMLTokenizer.scriptDataEscapedStatec C s | j j }|dkr2| jjtd dd | j| _n|dkrD| j| _nn|dkr| jjtd dd | jjtd dd | j| _n0|t kr| j
| _n | jjtd |d | j| _d S )
Nr| rJ )r# r$ rD r[ r" zinvalid-codepointu �T) r r6 r% r7 r r~ r r r r r )r r$ r r r! r { s"
z(HTMLTokenizer.scriptDataEscapedDashStatec C s | j j }|dkr*| jjtd dd n|dkr<| j| _n|dkrd| jjtd dd | j| _nn|dkr| jjtd dd | jjtd d d | j| _n0|t kr| j
| _n | jjtd |d | j| _d
S ) Nr| rJ )r# r$ rD rj r[ r" zinvalid-codepointu �T) r r6 r% r7 r r r rf r r r )r r$ r r r! r~ s&
z,HTMLTokenizer.scriptDataEscapedDashDashStatec C s | j j }|dkr"d| _| j| _n\|tkrT| jjtd d| d || _| j | _n*| jjtd dd | j j
| | j | _dS )Nri r, rJ rD )r# r$ T) r r6 rr scriptDataEscapedEndTagOpenStater r
r% r7 r scriptDataDoubleEscapeStartStater= r )r r$ r r r! r s
z0HTMLTokenizer.scriptDataEscapedLessThanSignStatec C sP | j j }|tkr"|| _| j| _n*| jjtd dd | j j | | j
| _dS )NrJ z</)r# r$ T) r r6 r
rr scriptDataEscapedEndTagNameStater r% r7 r r= r )r r$ r r r! r s
z.HTMLTokenizer.scriptDataEscapedEndTagOpenStatec C s | j o | j d j | jj k}| jj }|tkrT|rTtd | jg dd| _ | j| _n|dkr|rtd | jg dd| _ | j | _n||dkr|rtd | jg dd| _ | j
| j | _nH|t kr| j|7 _n0| j
jtd d| j d | jj| | j| _d
S ) NrU rV F)r# rU r$ rW ri rj rJ z</)r# r$ T)r ru rr r r6 r r rp r rq rZ r r
r% r7 r= r )r rv r$ r r r! r s2
z.HTMLTokenizer.scriptDataEscapedEndTagNameStatec C s | j j }|ttdB krR| jjtd |d | jj dkrH| j | _
q| j | _
nB|t kr| jjtd |d | j|7 _n| j j
| | j | _
dS )Nri rj rJ )r# r$ scriptT)ri rj )r r6 r r: r% r7 r rr ru scriptDataDoubleEscapedStater r r
r= )r r$ r r r! r s
z.HTMLTokenizer.scriptDataDoubleEscapeStartStatec C s | j j }|dkr2| jjtd dd | j| _n|dkrZ| jjtd dd | j| _nt|dkr| jjtd dd | jjtd dd n>|tkr| jjtd d d | j | _n| jjtd |d d
S ) Nr| rJ )r# r$ rD r[ r" zinvalid-codepointu �zeof-in-script-in-scriptT)
r r6 r% r7 r scriptDataDoubleEscapedDashStater (scriptDataDoubleEscapedLessThanSignStater r )r r$ r r r! r s$
z*HTMLTokenizer.scriptDataDoubleEscapedStatec C s | j j }|dkr2| jjtd dd | j| _n|dkrZ| jjtd dd | j| _n|dkr| jjtd dd | jjtd dd | j| _nF|t kr| jjtd d d | j
| _n | jjtd |d | j| _d
S ) Nr| rJ )r# r$ rD r[ r" zinvalid-codepointu �zeof-in-script-in-scriptT) r r6 r% r7 r $scriptDataDoubleEscapedDashDashStater r r r r )r r$ r r r! r s(
z.HTMLTokenizer.scriptDataDoubleEscapedDashStatec C s | j j }|dkr*| jjtd dd n|dkrR| jjtd dd | j| _n|dkrz| jjtd dd | j| _n|dkr| jjtd dd | jjtd d d | j| _nF|t kr| jjtd d
d | j
| _n | jjtd |d | j| _d S ) Nr| rJ )r# r$ rD rj r[ r" zinvalid-codepointu �zeof-in-script-in-scriptT) r r6 r% r7 r r r rf r r r )r r$ r r r! r s,
z2HTMLTokenizer.scriptDataDoubleEscapedDashDashStatec C sP | j j }|dkr8| jjtd dd d| _| j| _n| j j| | j | _dS )Nri rJ )r# r$ r, T)
r r6 r% r7 r rr scriptDataDoubleEscapeEndStater r= r )r r$ r r r! r 0 s
z6HTMLTokenizer.scriptDataDoubleEscapedLessThanSignStatec C s | j j }|ttdB krR| jjtd |d | jj dkrH| j | _
q| j | _
nB|t kr| jjtd |d | j|7 _n| j j
| | j | _
dS )Nri rj rJ )r# r$ r T)ri rj )r r6 r r: r% r7 r rr ru r r r r
r= )r r$ r r r! r ; s
z,HTMLTokenizer.scriptDataDoubleEscapeEndStatec C s0 | j j }|tkr$| j jtd n|tkrJ| jd j|dg | j| _n|dkr\| j n|dkrn| j
| _n|dkr| j jt d
d d | jd j|dg | j| _n|d
kr| j jt d
dd | jd jddg | j| _nF|t
kr| j jt d
dd | j| _n | jd j|dg | j| _dS )NTr$ r, rj ri '"rH rD r" z#invalid-character-in-attribute-name)r# r$ r[ zinvalid-codepointu �z#expected-attribute-name-but-got-eof)r r rH rD )r r6 r r^ r
r r7 attributeNameStater rZ rq r% r r r )r r$ r r r! rp K s6
z&HTMLTokenizer.beforeAttributeNameStatec C s | j j }d}d}|dkr&| j| _n0|tkr^| jd d d || j jtd 7 < d} n|dkrld}n|tkr~| j| _n|dkr| j | _n|d kr| j
j t d
d d | jd d d d
7 < d}n|dkr| j
j t d
dd | jd d d |7 < d}nH|t
kr8| j
j t d
dd | j| _n | jd d d |7 < d}|r| jd d d jt| jd d d< xP| jd d d D ]:\}}| jd d d |kr| j
j t d
dd P qW |r| j dS ) NTFrH r$ r r rj ri r[ r" zinvalid-codepoint)r# r$ u �r r rD z#invalid-character-in-attribute-namezeof-in-attribute-namezduplicate-attributerK rK )r r rD rK rK rK rK rK rK )r r6 beforeAttributeValueStater r
r r^ r afterAttributeNameStaterq r% r7 r r r rX r rZ )r r$ ZleavingThisStateZ emitTokenrU _r r r! r i sR
&
z HTMLTokenizer.attributeNameStatec C sF | j j }|tkr$| j jtd n |dkr8| j| _n
|dkrJ| j n|tkrp| jd j |dg | j
| _n|dkr| j | _n|dkr| j j t
d d d
| jd j d dg | j
| _n|dk r| j j t
d dd
| jd j |dg | j
| _nF|tkr&| j j t
d dd
| j| _n | jd j |dg | j
| _dS )NTrH rj r$ r, ri r[ r" zinvalid-codepoint)r# r$ u �r r rD z&invalid-character-after-attribute-namez expected-end-of-tag-but-got-eof)r r rD )r r6 r r^ r r rZ r
r r7 r rq r% r r r )r r$ r r r! r s:
z%HTMLTokenizer.afterAttributeNameStatec C sj | j j }|tkr$| j jtd nB|dkr8| j| _n.|dkrX| j| _| j j| n|dkrl| j| _ n|dkr| j j
t d dd | j n|d kr| j j
t d d
d | j
d d d d
7 < | j| _n|dkr| j j
t d dd | j
d d d |7 < | j| _nL|tkrD| j j
t d dd | j| _n"| j
d d d |7 < | j| _dS )NTr rC r rj r" z.expected-attribute-value-but-got-right-bracket)r# r$ r[ zinvalid-codepointr$ r u �rH rD `z"equals-in-unquoted-attribute-valuez$expected-attribute-value-but-got-eofrK )rH rD r rK rK )r r6 r r^ attributeValueDoubleQuotedStater attributeValueUnQuotedStater= attributeValueSingleQuotedStater% r7 r rZ r r r )r r$ r r r! r s>
z'HTMLTokenizer.beforeAttributeValueStatec C s | j j }|dkr | j| _n|dkr0| jd n|dkrj| jjtd dd | jd d d d 7 < nN|t kr| jjtd d
d | j
| _n&| jd d
d || j j d 7 < d S )Nr rC r[ r" zinvalid-codepoint)r# r$ r$ r u �z#eof-in-attribute-value-double-quoteTrK rK )r rC r[ ) r r6 afterAttributeValueStater rT r% r7 r r r r r^ )r r$ r r r! r s
z-HTMLTokenizer.attributeValueDoubleQuotedStatec C s | j j }|dkr | j| _n|dkr0| jd n|dkrj| jjtd dd | jd d d d 7 < nN|t kr| jjtd d
d | j
| _n&| jd d
d || j j d 7 < d S )Nr rC r[ r" zinvalid-codepoint)r# r$ r$ r u �z#eof-in-attribute-value-single-quoteTrK rK )r rC r[ ) r r6 r r rT r% r7 r r r r r^ )r r$ r r r! r s
z-HTMLTokenizer.attributeValueSingleQuotedStatec C s | j j }|tkr | j| _ n|dkr2| jd n|dkrD| j n|dkr~| jjt d d d
| j
d d d |7 < n|d
kr| jjt d dd
| j
d d d d7 < nV|t kr| jjt d dd
| j | _n.| j
d d d || j j
tdtB 7 < dS )NrC rj r r rH rD r r" z0unexpected-character-in-unquoted-attribute-value)r# r$ r$ r r[ zinvalid-codepointu �z eof-in-attribute-value-no-quotesT)r r rH rD r rK rK rK )rC rj r r rH rD r r[ )r r6 r rp r rT rZ r% r7 r r r r r^ r: )r r$ r r r! r s,
z)HTMLTokenizer.attributeValueUnQuotedStatec C s | j j }|tkr | j| _n|dkr.| j np|dkr@| j| _n^|tkrt| jj t
d dd | j j | | j | _n*| jj t
d dd | j j | | j| _dS )Nrj ri r" z$unexpected-EOF-after-attribute-value)r# r$ z*unexpected-character-after-attribute-valueT)
r r6 r rp r rZ rq r r% r7 r r= r )r r$ r r r! r s"
z&HTMLTokenizer.afterAttributeValueStatec C s | j j }|dkr&d| jd< | j n^|tkrZ| jjtd dd | j j| | j | _
n*| jjtd dd | j j| | j | _
dS )Nrj TrW r" z#unexpected-EOF-after-solidus-in-tag)r# r$ z)unexpected-character-after-solidus-in-tag) r r6 r rZ r r% r7 r r= r r rp )r r$ r r r! rq 4 s
z&HTMLTokenizer.selfClosingStartTagStatec C sD | j jd}|jdd}| jjtd |d | j j | j| _dS )Nrj r[ u �Comment)r# r$ T) r r^ replacer% r7 r r6 r r )r r$ r r r! ro F s
z HTMLTokenizer.bogusCommentStatec C s | j j g}|d dkrT|j| j j |d dkrPtd dd| _| j| _dS n|d d krd}x.d&D ]&}|j| j j |d' |krjd}P qjW |rtd dd d dd| _| j| _dS n|d( dkrH| jd k rH| jj j
rH| jj j
d) j | jj j krHd}x2d*D ]*}|j| j j |d+ |krd}P qW |rH| j
| _dS | jjtd dd x |rz| j j|j q`W | j| _dS ),Nr r| r r, )r# r$ TdDoOr@ CtTyYpPeEFZDoctype)r# rU publicIdsystemIdcorrect[Ar" zexpected-dashes-or-doctyperK rK rK )r r r r r@ r r r r r r r r r )r r r r r r rK rK rK )r r r r r r rK )r r6 r7 r r commentStartStater doctypeStater ZtreeZ openElements namespaceZdefaultNamespacecdataSectionStater% r= r' ro )r r? matchedexpectedr r r! rl U sR
z(HTMLTokenizer.markupDeclarationOpenStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d | jj| j | j| _nP|t kr| jjtd d
d | jj| j | j| _n| jd |7 < | j
| _d S ) Nr| r[ r" zinvalid-codepoint)r# r$ r$ u �rj zincorrect-commentzeof-in-commentT) r r6 commentStartDashStater r% r7 r r r r commentState)r r$ r r r! r s(
z HTMLTokenizer.commentStartStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d | jj| j | j| _nT|t kr| jjtd d
d | jj| j | j| _n | jd d| 7 < | j
| _d S ) Nr| r[ r" zinvalid-codepoint)r# r$ r$ u -�rj zincorrect-commentzeof-in-commentT) r r6 commentEndStater r% r7 r r r r r )r r$ r r r! r s(
z#HTMLTokenizer.commentStartDashStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < nT|tkr| jjtd dd | jj| j | j | _n | jd || j j
d
7 < d S ) Nr| r[ r" zinvalid-codepoint)r# r$ r$ u �zeof-in-commentT)r| r[ ) r r6 commentEndDashStater r% r7 r r r r r^ )r r$ r r r! r s
zHTMLTokenizer.commentStatec C s | j j }|dkr | j| _n|dkrV| jjtd dd | jd d7 < | j| _nT|t kr| jjtd dd | jj| j | j
| _n | jd d| 7 < | j| _d S )
Nr| r[ r" zinvalid-codepoint)r# r$ r$ u -�zeof-in-comment-end-dashT) r r6 r r r% r7 r r r r r )r r$ r r r! r s
z!HTMLTokenizer.commentEndDashStatec C s, | j j }|dkr*| jj| j | j| _n|dkrd| jjtd dd | jd d7 < | j| _n|dkr| jjtd d d | j | _n|d
kr| jjtd d d | jd |7 < nj|t
kr| jjtd d d | jj| j | j| _n4| jjtd d
d | jd d| 7 < | j| _dS )Nrj r[ r" zinvalid-codepoint)r# r$ r$ u --�rh z,unexpected-bang-after-double-dash-in-commentr| z,unexpected-dash-after-double-dash-in-commentzeof-in-comment-double-dashzunexpected-char-in-commentz--T) r r6 r% r7 r r r r r commentEndBangStater )r r$ r r r! r s6
z HTMLTokenizer.commentEndStatec C s | j j }|dkr*| jj| j | j| _n|dkrN| jd d7 < | j| _n|dkr| jjtd dd | jd d 7 < | j | _nT|t
kr| jjtd d
d | jj| j | j| _n | jd d| 7 < | j | _d S ) Nrj r| r$ z--!r[ r" zinvalid-codepoint)r# r$ u --!�z eof-in-comment-end-bang-stateT) r r6 r% r7 r r r r r r r )r r$ r r r! r s(
z!HTMLTokenizer.commentEndBangStatec C s | j j }|tkr | j| _nj|tkr\| jjtd dd d| j d< | jj| j | j
| _n*| jjtd dd | j j | | j| _dS )Nr" z!expected-doctype-name-but-got-eof)r# r$ Fr zneed-space-after-doctypeT) r r6 r beforeDoctypeNameStater r r% r7 r r r r= )r r$ r r r! r s
zHTMLTokenizer.doctypeStatec C s | j j }|tkrn|dkrT| jjtd dd d| jd< | jj| j | j| _n|dkr| jjtd dd d | jd
< | j | _nR|t
kr| jjtd d d d| jd< | jj| j | j| _n|| jd
< | j | _d S )
Nrj r" z+expected-doctype-name-but-got-right-bracket)r# r$ Fr r[ zinvalid-codepointu �rU z!expected-doctype-name-but-got-eofT) r r6 r r% r7 r r r r doctypeNameStater )r r$ r r r! r s.
z$HTMLTokenizer.beforeDoctypeNameStatec C s | j j }|tkr2| jd jt| jd< | j| _n|dkrh| jd jt| jd< | jj | j | j
| _n|dkr| jj t d dd | jd d7 < | j | _nh|t
kr| jj t d dd d | jd
< | jd jt| jd< | jj | j | j
| _n| jd |7 < d S ) NrU rj r[ r" zinvalid-codepoint)r# r$ u �zeof-in-doctype-nameFr T)r r6 r r rX r afterDoctypeNameStater r% r7 r r r r )r r$ r r r! r 6 s,
z HTMLTokenizer.doctypeNameStatec C sR | j j }|tkrn8|dkr8| jj| j | j| _n|tkrd| jd< | j j | | jjt
d dd | jj| j | j| _ n|d!krd }x$d'D ] }| j j }||krd}P qW |r| j | _d S nJ|d(krd }x(d.D ] }| j j }||k rd}P qW |r| j | _d S | j j | | jjt
d d d |id d| jd< | j
| _d S )/Nrj Fr r" zeof-in-doctype)r# r$ r r TuUbBlLiIr@ r sSr r r r r r mMz*expected-space-or-right-bracket-in-doctyper$ )r# r$ r. )r r r r r r r r r r r@ r )r r r r r )r r r r r r r r r r r r )r r r r r )r r6 r r% r7 r r r r r= r afterDoctypePublicKeywordState afterDoctypeSystemKeywordStatebogusDoctypeState)r r$ r r r r r! r O sT
z#HTMLTokenizer.afterDoctypeNameStatec C s | j j }|tkr | j| _n|d
krP| jjtd dd | j j| | j| _nT|t kr| jjtd dd d| j
d< | jj| j
| j | _n| j j| | j| _d S ) Nr r r" zunexpected-char-in-doctype)r# r$ zeof-in-doctypeFr T)r r ) r r6 r "beforeDoctypePublicIdentifierStater r% r7 r r= r r r )r r$ r r r! r s"
z,HTMLTokenizer.afterDoctypePublicKeywordStatec C s | j j }|tkrn|dkr0d| jd< | j| _n|dkrLd| jd< | j| _n|dkr| jjt d dd d | jd
< | jj| j | j
| _nh|t kr| jjt d d d d | jd
< | jj| j | j
| _n(| jjt d d d d | jd
< | j | _d
S )Nr r, r r rj r" zunexpected-end-of-doctype)r# r$ Fr zeof-in-doctypezunexpected-char-in-doctypeT)
r r6 r r (doctypePublicIdentifierDoubleQuotedStater (doctypePublicIdentifierSingleQuotedStater% r7 r r r r )r r$ r r r! r s4
z0HTMLTokenizer.beforeDoctypePublicIdentifierStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d d
| jd < | jj| j | j| _nR|t kr| jjtd d d d
| jd < | jj| j | j| _n| jd |7 < d
S )Nr r[ r" zinvalid-codepoint)r# r$ r u �rj zunexpected-end-of-doctypeFr zeof-in-doctypeT)
r r6 !afterDoctypePublicIdentifierStater r% r7 r r r r )r r$ r r r! r s*
z6HTMLTokenizer.doctypePublicIdentifierDoubleQuotedStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d d
| jd < | jj| j | j| _nR|t kr| jjtd d d d
| jd < | jj| j | j| _n| jd |7 < d
S )Nr r[ r" zinvalid-codepoint)r# r$ r u �rj zunexpected-end-of-doctypeFr zeof-in-doctypeT)
r r6 r r r% r7 r r r r )r r$ r r r! r s*
z6HTMLTokenizer.doctypePublicIdentifierSingleQuotedStatec C s | j j }|tkr | j| _n|dkr<| jj| j | j| _n|dkrn| jjt d dd d| jd< | j
| _n|dkr| jjt d dd d| jd< | j | _nh|t kr| jjt d d d d
| jd < | jj| j | j| _n(| jjt d dd d
| jd < | j
| _d S )
Nrj r r" zunexpected-char-in-doctype)r# r$ r, r r zeof-in-doctypeFr T)r r6 r -betweenDoctypePublicAndSystemIdentifiersStater r% r7 r r r (doctypeSystemIdentifierDoubleQuotedState(doctypeSystemIdentifierSingleQuotedStater r )r r$ r r r! r s6
z/HTMLTokenizer.afterDoctypePublicIdentifierStatec C s | j j }|tkrn|dkr4| jj| j | j| _n|dkrPd| jd< | j| _n|dkrld| jd< | j | _nh|t
kr| jjt d dd d | jd
< | jj| j | j| _n(| jjt d d d d | jd
< | j | _d S )
Nrj r r, r r r" zeof-in-doctype)r# r$ Fr zunexpected-char-in-doctypeT)
r r6 r r% r7 r r r r r r r r )r r$ r r r! r s.
z;HTMLTokenizer.betweenDoctypePublicAndSystemIdentifiersStatec C s | j j }|tkr | j| _n|d
krP| jjtd dd | j j| | j| _nT|t kr| jjtd dd d| j
d< | jj| j
| j | _n| j j| | j| _d S ) Nr r r" zunexpected-char-in-doctype)r# r$ zeof-in-doctypeFr T)r r ) r r6 r "beforeDoctypeSystemIdentifierStater r% r7 r r= r r r )r r$ r r r! r s"
z,HTMLTokenizer.afterDoctypeSystemKeywordStatec C s | j j }|tkrn|dkr0d| jd< | j| _n|dkrLd| jd< | j| _n|dkr| jjt d dd d | jd
< | jj| j | j
| _nh|t kr| jjt d d d d | jd
< | jj| j | j
| _n(| jjt d dd d | jd
< | j | _d S )
Nr r, r r rj r" zunexpected-char-in-doctype)r# r$ Fr zeof-in-doctypeT)
r r6 r r r r r r% r7 r r r r )r r$ r r r! r / s4
z0HTMLTokenizer.beforeDoctypeSystemIdentifierStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d d
| jd < | jj| j | j| _nR|t kr| jjtd d d d
| jd < | jj| j | j| _n| jd |7 < d
S )Nr r[ r" zinvalid-codepoint)r# r$ r u �rj zunexpected-end-of-doctypeFr zeof-in-doctypeT)
r r6 !afterDoctypeSystemIdentifierStater r% r7 r r r r )r r$ r r r! r L s*
z6HTMLTokenizer.doctypeSystemIdentifierDoubleQuotedStatec C s | j j }|dkr | j| _n|dkrN| jjtd dd | jd d7 < n|dkr| jjtd d d d
| jd < | jj| j | j| _nR|t kr| jjtd d d d
| jd < | jj| j | j| _n| jd |7 < d
S )Nr r[ r" zinvalid-codepoint)r# r$ r u �rj zunexpected-end-of-doctypeFr zeof-in-doctypeT)
r r6 r r r% r7 r r r r )r r$ r r r! r d s*
z6HTMLTokenizer.doctypeSystemIdentifierSingleQuotedStatec C s | j j }|tkrn~|dkr4| jj| j | j| _n^|tkrt| jjt d dd d| jd< | jj| j | j| _n | jjt d dd | j
| _dS ) Nrj r" zeof-in-doctype)r# r$ Fr zunexpected-char-in-doctypeT) r r6 r r% r7 r r r r r r )r r$ r r r! r | s
z/HTMLTokenizer.afterDoctypeSystemIdentifierStatec C sZ | j j }|dkr*| jj| j | j| _n,|tkrV| j j| | jj| j | j| _n dS )Nrj T) r r6 r% r7 r r r r r= )r r$ r r r! r s
z HTMLTokenizer.bogusDoctypeStatec C s g }xt|j | jjd |j | jjd | jj }|tkr@P q|d dd dkrl|d d d |d< P q|j | qW dj|}|jd}|dkrx&t|D ]}| jj t d d
d qW |j
dd }|r| jj t d
|d | j | _ dS )N]rj r z]]r, r[ r r" zinvalid-codepoint)r# r$ u �rJ TrK rK r rK )
r7 r r^ r6 r r9 countranger% r r r r )r r$ r6 Z nullCountr r r r! r s.
z HTMLTokenizer.cdataSectionState)N)NF)N__name__
__module__ __qualname____doc__r r) rB rS rT rZ r r\ rb r` rd rf rg r] rm rn ra rs rt rc rw rx re ry r{ rz r} r r r~ r r r r r r r r r rp r r r r r r r rq ro rl r r r r r r r r r r r r r r r r r r r r r r r
__classcell__r r )r r! r s H
P #
6 "- 3 r N)Z
__future__r r r Zpip._vendor.sixr r; collectionsr Z constantsr r r
r r r
r r r r Z _inputstreamr Z_trier rL objectr r r r r! <module> s