jbc @ s d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l m Z d d l
m Z d d l m
Z
d d l m Z d d l m Z m Z m Z d d l m Z d d l m Z e j e Z e j rd d l Z e j Z n d d l Z e j j Z e! Z" e! Z# d Z% d
Z& y d d l' m( Z( d d l) Z) e) j* d Z+ e( e+ j Z% e% e( d
k re, Z" n e( d e% k oe( d k n re, Z# n Wn e- k
rn Xd Z. d Z/ d Z0 d e1 f d YZ2 d e2 f d YZ3 d e1 f d YZ4 d e5 f d YZ6 d Z7 d d d d d d d d e, e, d d e! e, d d Z8 d d d d d d d d d d Z9 d e1 f d! YZ: d d" Z; d# Z< d S($ iN( t parsedate( t ENOENT( t partial( t count( t
exceptions( t urlparset
urlunparset quote( t log( t versiont REDACTED( t LooseVersiont requestss 0.8.8s 0.7.0s 1.0.0c C sR t t | d d } | d rH | d rH | d | d <d | d <n t | S( Nt schemet httpi i t ( t listR R ( t urlt
parsed_url( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt _cleanurl; s
c G s3 d } | } x | D] } | | | } q W| S( Nc S so t t | } | d } | r? | j d r? | d 7} n | t t | d d 7} | | d <t | S( Ni t /t safes /:( R R t endswitht urlquotet strR ( R t add_ont
url_parsedt path( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt combine_singleG s
( ( t baset add_onsR R R ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt combine_urlE s
c K s | j } | j d r( d | } n | j j d r | j d r_ t j d | n | t d } y( t | d } | j } Wd QXWnU t k
r } | j
} | j
t k r t } n t
d | d | d
d d | n Xt | d | St | | Sd S(
s0 Wrapper function around readurl to allow passing a file path as url.
When url is not a local file path, passthrough any kwargs to readurl.
In the case of parameter passthrough to readurl, default values for some
parameters. See: call-signature of readurl in this module for param docs.
R s file://%ss file://t datas' Unable to post data to file resource %st rbNt causet codet headersR t contents( t lstript
startswitht lowert gett LOGt warningt lent opent readt IOErrort errnoR t NOT_FOUNDt UrlErrort Nonet FileResponset readurl( R t kwargst file_patht fpR% t eR# ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt read_file_or_urlV s"
%t StringResponsec B s& e Z d d Z d Z d Z RS( i c C s( | | _ i | _ | | _ d | _ d S( N( R# R$ R% R3 R ( t selfR% R# ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt __init__v s c O s | j d k r t St S( Ni ( R# t Falset True( R< t argsR6 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt ok| s c C s | j j d S( Ns utf-8( R% t decode( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt __str__ s ( t __name__t
__module__R= RA RC ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR; u s R4 c B s e Z d d Z RS( i c C s# t j | | d | | | _ d S( NR# ( R; R= R ( R< R R% R# ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR= s ( RD RE R= ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR4 s t UrlResponsec B sb e Z d Z e d Z e d Z e d Z e d Z e d Z d Z
RS( c C s
| | _ d S( N( t _response( R< t response( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR= s c C s
| j j S( N( RG t content( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR% s c C s
| j j S( N( RG R ( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR s c C s@ d } | r d } n d | j k o/ | k n r8 t St Sd S( Ni, i i ( R# R? R> ( R< t redirects_okt upper( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyRA s c C s
| j j S( N( RG R$ ( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR$ s c C s
| j j S( N( RG t status_code( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR# s c C s
| j j S( N( RG t text( R< ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyRC s ( RD RE R= t propertyR% R R> RA R$ R# RC ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyRF s R2 c B s e Z d d d d Z RS( c C sY t j | t | | | _ | | _ | | _ | j d k rL i | _ n | | _ d S( N( R/ R= R R" R# R$ R3 R ( R< R" R# R$ R ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR= s N( RD RE R3 R= ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR2 s c C s i } t | j } | d k r | r t s@ t j d t q d | k rg | d rg | d | d <n
t | d <d | k r d | k r | d | d g | d <q d | k r t | d | d <q n | S( Nt httpssG SSL is not supported in requests v%s, cert. verification can not occur!t ca_certst verifyt cert_filet key_filet cert( R R
t SSL_ENABLEDR* R+ t _REQ_VERR? R ( R t ssl_detailst ssl_argsR
( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt
_get_ssl_args s
i i c sJ t | } i | d 6} | j t | | |
| d <| sT | rK d n d } n | | d <| d k r t t | d | d <n | d k r g } n t r i t d 6} | | d <n d
} | r t t | d
d
} n i d t j
d 6} r| j n | | s4 f d
} | } n | rG| | d <n | d k r\d } n g } x|
rtt n t d | D]} | | | d <i } x | j
D] \ } } | d k rqn | d k r@| r@g | D] } | j | r| ^ q} | rJt j | | | <x | D] } t | | | <q"WqJq| | | <qWy | rt j d | |
rod n | | | n | d k rt j } n | } | j | } Wd QX| r| j n t j d | | j t | j | d
t | SWqt j k
r.} t | t j r{t | d r{t | j! d r{| j" t# | d | j! j d | j! j$ d | n5 | j" t# | d | t% rt | t j& rPn | r| | j | d rPn |
r| d k s| d
| k r/| d k r/| rt j d | n t' j( | q/qXqW| rF| d n d S( s Wrapper around requests.Session to read the url and retry if necessary
:param url: Mandatory url to request.
:param data: Optional form data to post the URL. Will set request_method
to 'POST' if present.
:param timeout: Timeout in seconds to wait for a response
:param retries: Number of times to retry on exception if exception_cb is
None or exception_cb returns True for the exception caught. Default is
to fail with 0 retries on exception.
:param sec_between: Default 1: amount of seconds passed to time.sleep
between retries. None or -1 means don't sleep.
:param headers: Optional dict of headers to send during request
:param headers_cb: Optional callable returning a dict of values to send as
headers during request
:param headers_redact: Optional list of header names to redact from the log
:param ssl_details: Optional dict providing key_file, ca_certs, and
cert_file keys for use on in ssl connections.
:param check_status: Optional boolean set True to raise when HTTPError
occurs. Default: True.
:param allow_redirects: Optional boolean passed straight to Session.request
as 'allow_redirects'. Default: True.
:param exception_cb: Optional callable which accepts the params
msg and exception and returns a boolean True if retries are permitted.
:param session: Optional exiting requests.Session instance to reuse.
:param infinite: Bool, set True to retry indefinitely. Default: False.
:param log_req_resp: Set False to turn off verbose debug messages.
:param request_method: String passed as 'method' to Session.request.
Typically GET, or POST. Default: POST if data is provided, GET
otherwise.
R t allow_redirectst POSTt GETt methodi t timeoutt
store_cookiest configi s
Cloud-Init/%ss
User-Agentc s S( N( ( R ( R$ ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt _cb s R iR$ s' [%s/%s] open '%s' with %s configurationt infiniteNs( Read from %s (%s, %sb) after %s attemptsRH RL R# s1 Please wait %s seconds while we wait to try again() R t updateRY R3 t maxt floatt CONFIG_ENABLEDR> t intR t version_stringR t ranget itemsR) t copyt deepcopyR
R* t debugR t Sessiont requestt raise_for_statusRL R, RI RF R t RequestExceptiont
isinstancet HTTPErrort hasattrRH t appendR2 R$ RU t SSLErrort timet sleep( R R R^ t retriest sec_betweenR$ t
headers_cbt headers_redactRW t check_statusRZ t exception_cbt sessionRb t log_req_respt request_methodt req_argst
req_configt manual_triest def_headersRa t excpst it filtered_req_argst kt vt matched_headerst keyt sesst rR9 ( ( R$ s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR5 s #
% (
c
C s t j }
d d } | d k r- | } n d } d }
d } xt r| d k ri | | |
} n t |
d d } x| D]} t j } |
d k r | | |
r Pn | d k r | r | | |
| k r t |
| | } q n d } d } y | d k r | | } n i } t | d | d | d | d
t d | } | j sd | j } t t | d
| j d | j
d | } nS | j sd | j } t t | d
| j d | j
d | } n
| | j f SWnE t k
r } d | } | } n# t k
r-} d | } | } n Xt t j |
} | rTd | n d } d | | | | f } | | | r | d | d | q q W| | |
rPn |
d }
t
j d | t j | qE Wt d f S( sI
urls: a list of urls to try
max_wait: roughly the maximum time to wait before giving up
The max time is *actually* len(urls)*timeout as each url will
be tried once and given the timeout provided.
a number <= 0 will always result in only one try
timeout: the timeout provided to urlopen
status_cb: call method with string message when a url is not available
headers_cb: call method with single argument of url to get headers
for request.
headers_redact: a list of header names to redact from the log
exception_cb: call method with 2 arguments 'msg' (per status_cb) and
'exception', the exception that occurred.
sleep_time_cb: call method with 2 arguments (response, loop_n) that
generates the next sleep time.
request_method: indicate the type of HTTP request, GET, PUT, or POST
returns: tuple of (url, response contents), on failure, (False, None)
the idea of this routine is to wait for the EC2 metadata service to
come up. On both Eucalyptus and EC2 we have seen the case where
the instance hit the MD before the MD service was up. EC2 seems
to have permanently fixed this, though.
In openstack, the metadata service might be painfully slow, and
unable to avoid hitting a timeout of even up to 10 seconds or more
(LP: #894279) for a simple GET.
Offset those needs with the need to not hang forever (and block boot)
on a system where cloud-init is configured to look for EC2 Metadata
service but is not going to find one. It is possible that the instance
data host (169.254.169.254) may be firewalled off Entirely for a system,
meaning that the connection will block forever unless a timeout is set.
A value of None for max_wait will retry indefinitely.
c S s t j | d S( N( R* Rm ( t msgt exc( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt
log_status_cb s c S s0 | d k r t S| d k p/ t j | | k S( Ni ( R3 R> Rw ( t max_waitt
start_time( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt timeup s i i i R R$ R| R^ R} R s empty response [%s]R# R s bad status code [%s]s request error [%s]s unexpected error [%s]s %sst unlimiteds Calling '%s' failed [%s/%s]: %sR t exceptions1 Please wait %s seconds while we wait to try againN( Rw R3 R? Rg R5 R> R% R# R2 t
ValueErrorR$ RA t ExceptionR* Rm Rx ( t urlsR R^ t status_cbR{ R| t
sleep_timeR~ t
sleep_time_cbR R R R t loop_nRH R t nowt reasont url_excR$ R9 t
time_takent max_wait_strt
status_msg( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt wait_for_urlb sx &
t OauthUrlHelperc B sq e Z d d d d d d Z d Z d Z d Z d Z d Z d Z d Z
d Z d
Z RS( s /run/oauth_skew.jsonc C s | | _ | p d | _ | | _ | | _ | | _ t | _ d | _ | j | j | j f } t | su t | _ n t
| s t d n | j } | p i | _
d S( NR i sB all or none of token_key, token_secret, or consumer_key can be set( t consumer_keyt consumer_secrett token_keyt token_secrett skew_data_fileR? t _do_oautht skew_change_limitt anyR> t allR t read_skew_filet skew_data( R< R R R R R t requiredt old( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR= s c C sP | j rL t j j | j rL t | j d d } t j | SWd QXn d S( Nt modeR ( R t osR t isfileR- t jsont loadR3 ( R< R8 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR s c C sp | j s
d S| j } | d k r. i } n | | | <t | j d d } | j t j | Wd QXd S( NR t w( R R R3 R- t writeR t dumps( R< t hostt valuet curR8 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt update_skew_file s
c C s4 t | t o* | j d k p* | j d k s1 d Sd | j k rW t j d | j d S| j d } y t j t | } Wn' t k
r } t j d | | d SXt
| t j } t | j j
} | j j | d } t | | | j k r#| j | | t j d | | n | | j | <d S( Ni i t dates$ Missing header 'date' in %s responses# Failed to convert datetime '%s': %si s$ Setting oauth clockskew for %s to %d( Rr R2 R# R$ R* R+ Rw t mktimeR R Rg R R t netlocR R) t absR R ( R< R R R t remote_timeR9 t skewR t old_skew( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR~ s*
c C s | j s
i Sd } t | j } | j rZ | | j k rZ t t j | j | } n t d | d | j d | j d | j
d | j d | S( NR R R R R t timestamp( R R3 R R R Rg Rw t
oauth_headersR R R R ( R< R R R ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR{ s c C sK t | j | j d | d <t | j | j d | d <| | | S( NR{ R~ ( R t _headers_cbR) t
_exception_cb( R< t wrapped_funcR@ R6 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt _wrapped* s
c O s | j t | | S( N( R R ( R< R@ R6 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR 1 s c O s | j t | | S( N( R R5 ( R< R@ R6 ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR5 4 s c C s: d } z | r! | | | } n Wd | j | | X| S( N( R3 R~ ( R< t extra_exception_cbR R t ret( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR 7 s c C s5 i } | r | | } n | j | j | | S( N( Rc R{ ( R< t extra_headers_cbR R$ ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR @ s
N(
RD RE R3 R= R R R~ R{ R R R5 R R ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR s c C s y d d l j } Wn t k
r2 t d n X| rH t | } n d } | j | d | d | d | d | j d | } | j | \ } } }
| S( Nis oauth support is not availablet
client_secrett resource_owner_keyt resource_owner_secrett signature_methodR ( t oauthlib.oauth1t oauth1t ImportErrort NotImplementedErrorR R3 t Clientt SIGNATURE_PLAINTEXTt sign( R R R R R R R t clientt _urit signed_headerst _body( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyR H s
c C sL t | t s t S| j t k r& t S| j rH t | j t j rH t St S( s readurl exception_cb that will retry on NOT_FOUND and Timeout.
Returns False to raise the exception from readurl, True to retry.
( Rr R2 R> R# R1 R? R" R t Timeout( R R ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt retry_on_url_exc_ s (= Rk R R R t sixRw t email.utilsR R0 R t functoolsR t itertoolsR R t six.moves.urllib.parseR R R R t cloudinitR t loggingR t getLoggerRD R* t PY2t httplibR1 t http.clientR R R> RU Rf R3 RV R
t distutils.versionR t
pkg_resourcest get_distributiont _REQR? R R R R: t objectR; R4 RF R/ R2 RY R5 R R R R ( ( ( s8 /usr/lib/python2.7/site-packages/cloudinit/url_helper.pyt <module> sj (
! wn