]4 @ s^ d d l Z d d l Z d g Z Gd d d Z Gd d d Z Gd d d Z d S) NRobotFileParserc @ s e Z d Z d d d Z d d Z d d Z d d Z d
d Z d d
Z d d Z d d Z
d d Z d S)r c C s> g | _ d | _ d | _ d | _ | j | d | _ d S)NFr )entries
default_entry disallow_all allow_allset_url last_checked)selfurl r 0/opt/alt/python35/lib64/python3.5/robotparser.py__init__ s
zRobotFileParser.__init__c C s | j S)N)r )r
r r r
mtime s zRobotFileParser.mtimec C s d d l } | j | _ d S)Nr )timer )r
r r r r
modified( s zRobotFileParser.modifiedc C s5 | | _ t j j | d d \ | _ | _ d S)N )r urllibparseurlparsehostpath)r
r r r r
r 0 s zRobotFileParser.set_urlc C s y t j j | j } Wnm t j j k
r } zG | j d k rO d | _ n' | j d k rv | j d k rv d | _ WYd d } ~ Xn) X| j } | j
| j d j d S)N Ti i zutf-8)r r )
r Zrequesturlopenr error HTTPErrorcoder r readr decode
splitlines)r
ferrrawr r r
r 5 s zRobotFileParser.readc C s> d | j k r* | j d k r: | | _ n | j j | d S)N*)
useragentsr r append)r
entryr r r
_add_entryB s zRobotFileParser._add_entryc C s d } t } | j x| D]} | sr | d k rJ t } d } n( | d k rr | j | t } d } | j d } | d k r | d | } | j } | s q | j d d } t | d k r | d j j | d <t j j
| d j | d <| d d k r_| d k rB| j | t } | j j | d d } q | d d k r| d k r| j
j t | d d d } q | d d k r | d k r | j
j t | d d
d } q W| d k r| j | d S) Nr r #:z
user-agentZdisallowFZallowT)Entryr r) findstripsplitlenlowerr r unquoter& r' rulelinesRuleLine)r
linesstater( lineir r r
r K sL
zRobotFileParser.parsec C s | j r
d S| j r d S| j s' d St j j t j j | } t j j d d | j | j | j
| j f } t j j | } | s d } x- | j
D]" } | j | r | j | Sq W| j r | j j | Sd S)NFTr /)r r r r r r r3
urlunparser paramsZqueryZfragmentquoter
applies_to allowancer )r
useragentr Z
parsed_urlr( r r r
can_fetch s$ zRobotFileParser.can_fetchc C s d j d d | j D S)Nr c S s g | ] } t | d q S)
)str).0r( r r r
<listcomp> s z+RobotFileParser.__str__.<locals>.<listcomp>)joinr )r
r r r
__str__ s zRobotFileParser.__str__N) __name__
__module__ __qualname__r r r r r r) r rA rG r r r r
r s
4 c @ s4 e Z d Z d d Z d d Z d d Z d S)r5 c C sY | d k r | r d } t j j t j j | } t j j | | _ | | _ d S)Nr T)r r r; r r= r r? )r
r r? r r r
r s
zRuleLine.__init__c C s | j d k p | j | j S)Nr% )r
startswith)r
filenamer r r
r> s zRuleLine.applies_toc C s | j r d n d d | j S)NZAllowZDisallowz: )r? r )r
r r r
rG s zRuleLine.__str__N)rH rI rJ r r> rG r r r r
r5 s r5 c @ s@ e Z d Z d d Z d d Z d d Z d d Z d S)
r- c C s g | _ g | _ d S)N)r& r4 )r
r r r
r s zEntry.__init__c C sj g } x' | j D] } | j d | d g q Wx* | j D] } | j t | d g q: Wd j | S)Nz User-agent: rB r )r& extendr4 rC rF )r
retagentr8 r r r
rG s z
Entry.__str__c C s] | j d d j } x= | j D]2 } | d k r9 d S| j } | | k r# d Sq# Wd S)Nr: r r% TF)r0 r2 r& )r
r@ rO r r r
r> s zEntry.applies_toc C s. x' | j D] } | j | r
| j Sq
Wd S)NT)r4 r> r? )r
rL r8 r r r
r? s zEntry.allowanceN)rH rI rJ r rG r> r? r r r r
r- s
r- )Z urllib.parser Zurllib.request__all__r r5 r- r r r r
<module>
s