Viewing File: /usr/lib/python3.6/site-packages/pip/_vendor/html5lib/__pycache__/_inputstream.cpython-36.pyc

3

aj)@sddlmZmZmZddlmZmZddlmZm	Z	ddl
Z
ddlZddlm
Z
ddlmZmZmZmZddlmZdd	lmZdd
lmZyddlmZWnek
reZYnXedd
eDZedd
eDZedd
eDZeeddgBZdZej rJedFdkr&ej!ddks*t"ej#eddGe$ddZ%n
ej#eZ%e&dddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4g Z'ej#d5Z(iZ)Gd6d7d7e*Z+d8d9Z,Gd:d;d;e*Z-Gd<d=d=e-Z.Gd>d?d?e/Z0Gd@dAdAe*Z1GdBdCdCe*Z2dDdEZ3dS)H)absolute_importdivisionunicode_literals)	text_typebinary_type)http_clienturllibN)webencodings)EOFspaceCharactersasciiLettersasciiUppercase)ReparseException)_utils)StringIO)BytesIOcCsg|]}|jdqS)ascii)encode).0itemr"/usr/lib/python3.6/_inputstream.py
<listcomp>srcCsg|]}|jdqS)r)r)rrrrrrscCsg|]}|jdqS)r)r)rrrrrrs><u[---Ÿ﷐-﷯￾￿🿾🿿𯿾𯿿𿿾𿿿񏿾񏿿񟿾񟿿񯿾񯿿񿿾񿿿򏿾򏿿򟿾򟿿򯿾򯿿򿿾򿿿󏿾󏿿󟿾󟿿󯿾󯿿󿿾󿿿􏿾􏿿]]z"\uD800-\uDFFF"iiiiiiiiiiiiiiiii	i	i
i
iiiii
i
iiiiiiz[	-
 -/:-@[-`{-~]c@sHeZdZdZddZddZddZdd	Zd
dZdd
Z	ddZ
dS)BufferedStreamzBuffering for streams that do not have buffering of their own

    The buffer is implemented as a list of chunks on the assumption that
    joining many strings will be slow since it is O(n**2)
    cCs||_g|_ddg|_dS)Nr
r)streambufferposition)selfrrrr__init__@szBufferedStream.__init__cCs@d}x(|jd|jdD]}|t|7}qW||jd7}|S)Nrr
)r r!len)r"poschunkrrrtellEs
zBufferedStream.tellcCsX||jkst|}d}x0t|j||krH|t|j|8}|d7}qW||g|_dS)Nrr
)_bufferedBytesAssertionErrorr$r r!)r"r%offsetirrrseekLszBufferedStream.seekcCsT|js|j|S|jdt|jkrF|jdt|jdkrF|j|S|j|SdS)Nrr
r)r _readStreamr!r$_readFromBuffer)r"bytesrrrreadUs

zBufferedStream.readcCstdd|jDS)NcSsg|]}t|qSr)r$)rrrrrr_sz1BufferedStream._bufferedBytes.<locals>.<listcomp>)sumr )r"rrrr(^szBufferedStream._bufferedBytescCs<|jj|}|jj||jdd7<t||jd<|S)Nrr
)rr0r appendr!r$)r"r/datarrrr-as
zBufferedStream._readStreamcCs|}g}|jd}|jd}x|t|jkr|dkr|dks@t|j|}|t||krn|}|||g|_n"t||}|t|g|_|d7}|j||||||8}d}qW|r|j|j|dj|S)Nrr
)r!r$r r)r2r-join)r"r/ZremainingBytesrvZbufferIndexZbufferOffsetZbufferedDataZbytesToReadrrrr.hs&


zBufferedStream._readFromBufferN)__name__
__module____qualname____doc__r#r'r,r0r(r-r.rrrrr9s		rcKst|tjs(t|tjjr.t|jtjr.d}n&t|drJt|jdt	}n
t|t	}|rdd|D}|rvt
d|t|f|St|f|SdS)NFr0rcSsg|]}|jdr|qS)Z	_encoding)endswith)rxrrrrsz#HTMLInputStream.<locals>.<listcomp>z3Cannot set an encoding with a unicode input, set %r)
isinstancerZHTTPResponserZresponseZaddbasefphasattrr0r	TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargsZ	isUnicodeZ	encodingsrrrHTMLInputStreams

rEc@speZdZdZdZddZddZddZd	d
ZddZ	d
dZ
dddZddZddZ
dddZddZdS)rAzProvides a unicode stream of characters to the HTMLTokenizer.

    This class takes care of character encoding and removing or replacing
    incorrect byte-sequences and also provides column and line tracking.

    i(cCsZtjsd|_ntddkr$|j|_n|j|_dg|_tddf|_|j	||_
|jdS)aInitialises the HTMLInputStream.

        HTMLInputStream(source, [encoding]) -> Normalized stream from source
        for use by html5lib.

        source can be either a file-object, local filename or a string.

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)

        Nu􏿿r
rzutf-8certain)rsupports_lone_surrogatesreportCharacterErrorsr$characterErrorsUCS4characterErrorsUCS2ZnewLineslookupEncodingcharEncoding
openStream
dataStreamreset)r"rCrrrr#s
zHTMLUnicodeInputStream.__init__cCs.d|_d|_d|_g|_d|_d|_d|_dS)Nr)r&	chunkSizechunkOffseterrorsprevNumLinesprevNumCols_bufferedCharacter)r"rrrrOszHTMLUnicodeInputStream.resetcCst|dr|}nt|}|S)zvProduces a file object from source.

        source can be either a file object, local filename or a string.

        r0)r?r)r"rCrrrrrMs
z!HTMLUnicodeInputStream.openStreamcCsT|j}|jdd|}|j|}|jdd|}|dkr@|j|}n||d}||fS)N
rr
r)r&countrTrfindrU)r"r*r&ZnLinesZpositionLineZlastLinePosZpositionColumnrrr	_positions
z HTMLUnicodeInputStream._positioncCs|j|j\}}|d|fS)z:Returns (line, col) of the current position in the stream.r
)rZrR)r"linecolrrrr!szHTMLUnicodeInputStream.positioncCs6|j|jkr|jstS|j}|j|}|d|_|S)zo Read one character from the stream or queue if available. Return
            EOF when EOF is reached.
        r
)rRrQ	readChunkrr&)r"rRcharrrrr^s

zHTMLUnicodeInputStream.charNcCs|dkr|j}|j|j\|_|_d|_d|_d|_|jj|}|j	rX|j	|}d|_	n|s`dSt
|dkrt|d}|dksd|kodknr|d
|_	|dd}|jr|j||j
dd	}|j
d
d	}||_t
||_dS)NrPrFr

iiz
rW
Trrr)_defaultChunkSizerZrQrTrUr&rRrNr0rVr$ordrHreplace)r"rQr3Zlastvrrrr]s0
 


z HTMLUnicodeInputStream.readChunkcCs,x&tttj|D]}|jjdqWdS)Nzinvalid-codepoint)ranger$invalid_unicode_refindallrSr2)r"r3_rrrrI%sz*HTMLUnicodeInputStream.characterErrorsUCS4cCsd}xtj|D]}|rqt|j}|j}tj|||drttj|||d}|tkrn|j	j
dd}q|dkr|dkr|t|dkr|j	j
dqd}|j	j
dqWdS)NFzinvalid-codepointTiir
)refinditerrbgroupstartrZisSurrogatePairZsurrogatePairToCodepointnon_bmp_invalid_codepointsrSr2r$)r"r3skipmatchZ	codepointr%Zchar_valrrrrJ)s z*HTMLUnicodeInputStream.characterErrorsUCS2Fc
Csyt||f}Wnltk
r|x|D]}t|dks&tq&Wdjdd|D}|s^d|}tjd|}t||f<YnXg}x||j|j|j	}|dkr|j	|j
krPn0|j}||j
kr|j|j|j	|||_	P|j|j|j	d|j
sPqWdj|}	|	S)z Returns a string of characters from the stream up to but not
        including any character in 'characters' or EOF. 'characters' must be
        a container that supports the 'in' method and iteration over its
        characters.
        rPcSsg|]}dt|qS)z\x%02x)rb)rcrrrrNsz5HTMLUnicodeInputStream.charsUntil.<locals>.<listcomp>z^%sz[%s]+N)charsUntilRegExKeyErrorrbr)r5recompilernr&rRrQendr2r])
r"Z
charactersZoppositecharsrpZregexr6mrurrrr
charsUntil@s2
 

z!HTMLUnicodeInputStream.charsUntilcCsT|dk	rP|jdkr.||j|_|jd7_n"|jd8_|j|j|ksPtdS)Nrr
)rRr&rQr))r"r^rrrungetos
zHTMLUnicodeInputStream.unget)N)F)r7r8r9r:rar#rOrMrZr!r^r]rIrJryrzrrrrrAs 
&
/rAc@sLeZdZdZdddZddZd	d
ZdddZd
dZddZ	ddZ
dS)rBzProvides a unicode stream of characters to the HTMLTokenizer.

    This class takes care of character encoding and removing or replacing
    incorrect byte-sequences and also provides column and line tracking.

    Nwindows-1252TcCsn|j||_tj||jd|_d|_||_||_||_||_	||_
|j||_|jddk	sbt
|jdS)aInitialises the HTMLInputStream.

        HTMLInputStream(source, [encoding]) -> Normalized stream from source
        for use by html5lib.

        source can be either a file-object, local filename or a string.

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)

        idrN)rM	rawStreamrAr#numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingrLr)rO)r"rCrrrrrZ
useChardetrrrr#szHTMLBinaryInputStream.__init__cCs&|jdjj|jd|_tj|dS)Nrrc)rLZ
codec_infostreamreaderr}rNrArO)r"rrrrOszHTMLBinaryInputStream.resetc	CsDt|dr|}nt|}y|j|jWnt|}YnX|S)zvProduces a file object from source.

        source can be either a file object, local filename or a string.

        r0)r?rr,r'r)r"rCrrrrrMs
z HTMLBinaryInputStream.openStreamcCs|jdf}|ddk	r|St|jdf}|ddk	r:|St|jdf}|ddk	rX|S|jdf}|ddk	rt|St|jdf}|ddk	r|djjdr|St|jdf}|ddk	r|S|rtyddl	m
}Wntk
rYnXg}|}xF|js>|j
j|j}t|ts t|s(P|j||j|qW|jt|jd}|j
jd|dk	rt|dfSt|jdf}|ddk	r|StddfS)NrFrZ	tentativezutf-16)UniversalDetectorencodingzwindows-1252)	detectBOMrKrrdetectEncodingMetarname
startswithrZchardet.universaldetectorrImportErrordoner}r0rr=r/r)r2Zfeedcloseresultr,r)r"ZchardetrLrZbuffersZdetectorr rrrrrsR


z'HTMLBinaryInputStream.determineEncodingcCs|jddkstt|}|dkr&dS|jdkrFtd}|dk	stnT||jdkrf|jddf|_n4|jjd|df|_|jtd|jd|fdS)	Nr
rFutf-16beutf-16lezutf-8rzEncoding changed from %s to %s)rr)rLr)rKrr}r,rOr)r"ZnewEncodingrrrchangeEncodings

z$HTMLBinaryInputStream.changeEncodingc
Cstjdtjdtjdtjdtjdi}|jjd}t|t	s<t
|j|dd}d}|s~|j|}d}|s~|j|dd	}d	}|r|jj|t
|S|jjd
dSdS)zAttempts to detect at BOM at the start of the stream. If
        an encoding can be determined from the BOM return the name of the
        encoding otherwise return Nonezutf-8zutf-16lezutf-16bezutf-32lezutf-32beNrhr)codecsBOM_UTF8BOM_UTF16_LEBOM_UTF16_BEBOM_UTF32_LEBOM_UTF32_BEr}r0r=r/r)getr,rK)r"ZbomDictstringrr,rrrrs$
zHTMLBinaryInputStream.detectBOMcCsV|jj|j}t|tstt|}|jjd|j}|dk	rR|j	dkrRt
d}|S)z9Report the encoding declared by the meta element
        rNutf-16beutf-16lezutf-8)rr)r}r0r~r=r/r)EncodingParserr,getEncodingrrK)r"r parserrrrrr9sz(HTMLBinaryInputStream.detectEncodingMeta)NNNNr{T)T)r7r8r9r:r#rOrMrrrrrrrrrBs
(
>"rBc@seZdZdZddZddZddZdd	Zd
dZdd
Z	ddZ
ddZeee
Z
ddZeeZefddZddZddZddZdS)
EncodingByteszString-like object with an associated position and various extra methods
    If the position is ever greater than the string length then an exception is
    raisedcCst|tsttj||jS)N)r=r/r)__new__lower)r"valuerrrrLszEncodingBytes.__new__cCs
d|_dS)Nr
r)rZ)r"rrrrr#PszEncodingBytes.__init__cCs|S)Nr)r"rrr__iter__TszEncodingBytes.__iter__cCs>|jd}|_|t|kr"tn|dkr.t|||dS)Nr
r)rZr$
StopIterationr@)r"prrr__next__WszEncodingBytes.__next__cCs|jS)N)r)r"rrrnext_szEncodingBytes.nextcCsB|j}|t|krtn|dkr$t|d|_}|||dS)Nrr
)rZr$rr@)r"rrrrpreviouscszEncodingBytes.previouscCs|jt|krt||_dS)N)rZr$r)r"r!rrrsetPositionlszEncodingBytes.setPositioncCs*|jt|krt|jdkr"|jSdSdS)Nr)rZr$r)r"rrrgetPositionqs

zEncodingBytes.getPositioncCs||j|jdS)Nr
)r!)r"rrrgetCurrentByte{szEncodingBytes.getCurrentBytecCsL|j}x:|t|kr@|||d}||kr6||_|S|d7}qW||_dS)zSkip past a list of charactersr
N)r!r$rZ)r"rvrrprrrrmszEncodingBytes.skipcCsL|j}x:|t|kr@|||d}||kr6||_|S|d7}qW||_dS)Nr
)r!r$rZ)r"rvrrprrr	skipUntilszEncodingBytes.skipUntilcCs>|j}|||t|}|j|}|r:|jt|7_|S)zLook for a sequence of bytes at the start of a string. If the bytes
        are found return True and advance the position to the byte after the
        match. Otherwise return False and leave the position alone)r!r$r)r"r/rr3r6rrr
matchBytess
zEncodingBytes.matchBytescCsR||jdj|}|dkrJ|jdkr,d|_|j|t|d7_dStdS)zLook for the next sequence of bytes matching a given sequence. If
        a match is found advance the position to the last byte of the matchNr
rTrr)r!findrZr$r)r"r/ZnewPositionrrrjumpTos
zEncodingBytes.jumpToN)r7r8r9r:rr#rrrrrrpropertyr!rcurrentBytespaceCharactersBytesrmrrrrrrrrHs 	
rc@sXeZdZdZddZddZddZdd	Zd
dZdd
Z	ddZ
ddZddZdS)rz?Mini parser for detecting character encoding from meta elementscCst||_d|_dS)z3string - the data to work on for encoding detectionN)rr3r)r"r3rrrr#s
zEncodingParser.__init__c
Csd|jfd|jfd|jfd|jfd|jfd|jff}x^|jD]T}d}xD|D]<\}}|jj|rJy|}PWqJtk
rd}PYqJXqJW|s<Pq<W|jS)	Ns<!--s<metas</s<!s<?rTF)	
handleComment
handleMetahandlePossibleEndTaghandleOtherhandlePossibleStartTagr3rrr)r"ZmethodDispatchrgZkeepParsingkeymethodrrrrs&zEncodingParser.getEncodingcCs|jjdS)zSkip over commentss-->)r3r)r"rrrrszEncodingParser.handleCommentcCs|jjtkrdSd}d}x|j}|dkr.dS|ddkr^|ddk}|r|dk	r||_dSq|ddkr|d}t|}|dk	r||_dSq|ddkrtt|d}|j}|dk	rt|}|dk	r|r||_dS|}qWdS)	NTFrs
http-equivr
scontent-typescharsetscontent)	r3rrgetAttributerrKContentAttrParserrparse)r"Z	hasPragmaZpendingEncodingattrZtentativeEncodingcodecZ
contentParserrrrrs:zEncodingParser.handleMetacCs
|jdS)NF)handlePossibleTag)r"rrrrsz%EncodingParser.handlePossibleStartTagcCst|j|jdS)NT)rr3r)r"rrrrs
z#EncodingParser.handlePossibleEndTagcCsf|j}|jtkr(|r$|j|jdS|jt}|dkrD|jn|j}x|dk	r`|j}qNWdS)NTr)r3rasciiLettersBytesrrrspacesAngleBracketsr)r"ZendTagr3rprrrrrs



z EncodingParser.handlePossibleTagcCs|jjdS)Nr)r3r)r"rrrrszEncodingParser.handleOthercCs|j}|jttdgB}|dks2t|dks2t|d	kr>dSg}g}xt|dkrX|rXPnX|tkrl|j}PnD|d
krdj|dfS|tkr|j|j	n|dkrdS|j|t
|}qHW|dkr|jdj|dfSt
||j}|dkrT|}xt
|}||kr(t
|dj|dj|fS|tkrB|j|j	n
|j|qWnJ|dkrldj|dfS|tkr|j|j	n|dkrdS|j|x^t
|}|tkrdj|dj|fS|tkr|j|j	n|dkrdS|j|qWdS)z_Return a name,value pair for the next attribute in the stream,
        if one is found, or None/Nr
r=r4'")rN)rr)rr)
r3rmr	frozensetr$r)r5asciiUppercaseBytesr2rrrr)r"r3rpZattrNameZ	attrValueZ	quoteCharrrrrsh










zEncodingParser.getAttributeN)
r7r8r9r:r#rrrrrrrrrrrrrs$rc@seZdZddZddZdS)rcCst|tst||_dS)N)r=r/r)r3)r"r3rrrr#fszContentAttrParser.__init__cCsy|jjd|jjd7_|jj|jjdks8dS|jjd7_|jj|jjdkr|jj}|jjd7_|jj}|jj|r|j||jjSdSnF|jj}y|jjt|j||jjStk
r|j|dSXWntk
rdSXdS)Nscharsetr
rrr)rr)r3rr!rmrrrr)r"Z	quoteMarkZoldPositionrrrrjs.

zContentAttrParser.parseN)r7r8r9r#rrrrrresrcCs`t|tr.y|jd}Wntk
r,dSX|dk	rXy
tj|Stk
rTdSXndSdS)z{Return the python codec name corresponding to an encoding or None if the
    string doesn't correspond to a valid encoding.rN)r=rdecodeUnicodeDecodeErrorr	lookupAttributeError)rrrrrKs

rKrr)4Z
__future__rrrZpip._vendor.sixrrZpip._vendor.six.movesrrrrsZpip._vendorr	Z	constantsrrr
rrrPriorrrrrrrrZinvalid_unicode_no_surrogaterGrXr)rtevalresetrlZascii_punctuation_rerqobjectrrErArBr/rrrrKrrrr<module>sX
"








JgIh6'
Back to Directory File Manager