­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 ctj~)@sddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z*m+Z+y ddl,Z,Wne-k r"dZ.YnXdZ.ddd d d d d ddddddddddddddddddd d!d"d#d$d%d&d'g!Z/d(e j0dd)Z1da2de j3fddddd*d+dZ4d,dZ5gZ6d}d-d$Z7d.d%Z8e j9d/e j:Z;d0d1Zd4d Z?Gd5d d Z@Gd6dde@ZAGd7d d e@ZBGd8d d e@ZCd9d:ZDGd;d d e@ZEGdddeGZHGd?ddZIGd@ddeIe@ZJGdAddeIe@ZKejLZMGdBddZNGdCdde@eNZOGdDdde@eNZPGdEdFdFe@ZQGdGddeQZReSejTdHr.GdIdJdJeQZUe/jVdJGdKd d e@ZWGdLdde@ZXdMdNZYdOdPZZGdQdde@Z[dRdSZ\GdTdde@Z]GdUdde]Z^GdVdde@Z_dWZ`ejadXkrddYlbmcZcmdZdndZd"Zcd[d!ZdiZeGd\d&d&ZfGd]d'd'efZgdahd^d_Zidajd`daZkdaldbdcZmdandddeZoGdfdgdgZpdhdiZqd~djdkZrdldmZse jtdnkrddolumvZvmwZwdpdqZxdrdsZydtduZzdvd#Z{n6ejadXkrdwdxZ|dyd#Z{dzd{Z}d|duZzneqZ{erZzdS)N)URLError HTTPErrorContentTooShortError)urlparseurlspliturljoinunwrapquoteunquote splittype splithost splitport splituser splitpasswd splitattr splitquery splitvaluesplittagto_bytesunquote_to_bytes urlunparse) addinfourl addclosehookFTRequestOpenerDirector BaseHandlerHTTPDefaultErrorHandlerHTTPRedirectHandlerHTTPCookieProcessor ProxyHandlerHTTPPasswordMgrHTTPPasswordMgrWithDefaultRealmHTTPPasswordMgrWithPriorAuthAbstractBasicAuthHandlerHTTPBasicAuthHandlerProxyBasicAuthHandlerAbstractDigestAuthHandlerHTTPDigestAuthHandlerProxyDigestAuthHandler HTTPHandler FileHandler FTPHandlerCacheFTPHandler DataHandlerUnknownHandlerHTTPErrorProcessorurlopeninstall_opener build_opener pathname2url url2pathname getproxies urlretrieve urlcleanup URLopenerFancyURLopenerz%d.%d)cafilecapath cadefaultcontextc Cs|s |s |rfddl}|jdtd|dk r2tdts>tdtjtjj||d}t |d}t |} n0|r~t |d}t |} nt dkrt a } nt } | j |||S)NrzJcafile, capath and cadefault are deprecated, use a custom context instead.r:zDYou can't pass both context and any of cafile, capath, and cadefaultzSSL support not available)r;r<)r>) warningswarnDeprecationWarning ValueError _have_sslsslZcreate_default_contextZPurposeZ SERVER_AUTH HTTPSHandlerr2_openeropen) urldatatimeoutr;r<r=r>r?Z https_handleropenerrL&/usr/lib64/python3.6/urllib/request.pyr0s*<       cCs|adS)N)rF)rKrLrLrMr1scCs4t|\}}tjt||}|j}|dkrD| rDtjj||fS|rTt|d}nt j dd}|j }t j ||||f} d } d } d} d} d |krt|d } |r|| | | xB|j| }|sP| t|7} |j|| d7} |r|| | | qWWdQRXWdQRX| dkr0| | kr0td | | f| | S)NfilewbF)deleteirzcontent-lengthzContent-Lengthz1retrieval incomplete: got only %i out of %i bytesi )r contextlibclosingr0infoospathnormpathrGtempfileZNamedTemporaryFilename_url_tempfilesappendintreadlenwriter)rHfilename reporthookrIZurl_typerXfpheaderstfpresultbssizer_blocknumblockrLrLrMr6sD         $c CsHx0tD](}ytj|Wqtk r,YqXqWtdd=trDdadS)N)r\rWunlinkOSErrorrF)Z temp_filerLrLrMr7%s   z:\d+$cCs<|j}t|d}|dkr&|jdd}tjd|d}|jS)NrRHost)full_urlr get_header _cut_port_resublower)requestrHhostrLrLrM request_host4s   rwc@seZdZdidddfddZeddZejddZejddZed d Zejd d Zejd d Zd dZ ddZ ddZ ddZ ddZ ddZddZddZd#ddZdd Zd!d"ZdS)$rNFc Csp||_i|_i|_d|_||_d|_x |jD]\}}|j||q.W|dkrVt|}||_ ||_ |rl||_ dS)N) rpreunredirected_hdrs_datarI _tunnel_hostitems add_headerrworigin_req_host unverifiablemethod) selfrHrIrer}r~rkeyvaluerLrLrM__init__FszRequest.__init__cCs|jrdj|j|jS|jS)Nz{}#{})fragmentformat _full_url)rrLrLrMrpXszRequest.full_urlcCs(t||_t|j\|_|_|jdS)N)rrrr_parse)rrHrLrLrMrp^s cCsd|_d|_d|_dS)Nrn)rrselector)rrLrLrMrpescCs|jS)N)ry)rrLrLrMrIksz Request.datacCs(||jkr$||_|jdr$|jddS)NzContent-length)ry has_header remove_header)rrIrLrLrMrIos  cCs d|_dS)N)rI)rrLrLrMrIyscCsNt|j\|_}|jdkr(td|jt|\|_|_|jrJt|j|_dS)Nzunknown url type: %r) r rtyperBrpr rvrr )rrestrLrLrMr}s  zRequest._parsecCs|jdk rdnd}t|d|S)NPOSTGETr)rIgetattr)rZdefault_methodrLrLrM get_methodszRequest.get_methodcCs|jS)N)rp)rrLrLrM get_full_urlszRequest.get_full_urlcCs4|jdkr|j r|j|_n||_|j|_||_dS)Nhttps)rrzrvrpr)rrvrrLrLrM set_proxys  zRequest.set_proxycCs |j|jkS)N)rrp)rrLrLrM has_proxyszRequest.has_proxycCs||j|j<dS)N)re capitalize)rrvalrLrLrMr|szRequest.add_headercCs||j|j<dS)N)rxr)rrrrLrLrMadd_unredirected_headerszRequest.add_unredirected_headercCs||jkp||jkS)N)rerx)r header_namerLrLrMrs zRequest.has_headercCs|jj||jj||S)N)regetrx)rrdefaultrLrLrMrqszRequest.get_headercCs |jj|d|jj|ddS)N)repoprx)rrrLrLrMrszRequest.remove_headercCs"|jj}|j|jt|jS)N)rxcopyupdaterelistr{)rhdrsrLrLrM header_itemss  zRequest.header_items)N)__name__ __module__ __qualname__rpropertyrpsetterdeleterrIrrrrrr|rrrqrrrLrLrLrMrDs(     c@sNeZdZddZddZddZddZd ejfd d Z dd d Z ddZ d S)rcCs6dt}d|fg|_g|_i|_i|_i|_i|_dS)NzPython-urllib/%sz User-agent) __version__ addheadershandlers handle_open handle_errorprocess_responseprocess_request)rZclient_versionrLrLrMrs zOpenerDirector.__init__c CsZt|dstdt|d}xt|D]}|dkr:q*|jd}|d|}||dd}|jd r|jd|d}||dd}y t|}Wntk rYnX|jj |i} | |j|<n>|d kr|}|j } n*|d kr|}|j } n|d kr*|}|j } nq*| j |g} | r&tj| |n | j|d }q*W|rVtj|j||j|dS)N add_parentz%expected BaseHandler instance, got %rFredirect_requestdo_open proxy_open_rRerrorrGresponseruT)rrr)hasattr TypeErrorrdirfind startswithr^rBrrrrr setdefaultbisectZinsortr]rr) rhandlerZaddedmethiprotocolZ conditionjkindlookuprrLrLrM add_handlersJ         zOpenerDirector.add_handlercCsdS)NrL)rrLrLrMcloseszOpenerDirector.closec Gs<|j|f}x*|D]"}t||}||}|dk r|SqWdS)N)rr) rchainr meth_nameargsrrfuncrgrLrLrM _call_chains    zOpenerDirector._call_chainNc Cst|trt||}n|}|dk r(||_||_|j}|d}x(|jj|gD]}t||}||}qLW|j ||} |d}x*|j j|gD]}t||}||| } qW| S)NZ_requestZ _response) isinstancestrrrIrJrrrr_openr) rfullurlrIrJreqrrZ processorrrrLrLrMrGs"      zOpenerDirector.opencCsP|j|jdd|}|r|S|j}|j|j||d|}|r>|S|j|jdd|S)NrZ default_openrunknown unknown_open)rrr)rrrIrgrrLrLrMrs    zOpenerDirector._opencGs~|d kr,|jd}|d}d|}d}|}n|j}|d}d}|||f|}|j|}|r^|S|rz|dd f|}|j|SdS) Nhttprr:z http_error_%srRZ_errorrrhttp_error_default)rr)rr)rprotordictrZhttp_errZ orig_argsrgrLrLrMr's   zOpenerDirector.error)N) rrrrrrrsocket_GLOBAL_DEFAULT_TIMEOUTrGrrrLrLrLrMrs /  c Gst}ttttttttt g }t t j dr2|j tt}xN|D]F}x@|D]8}t|trlt||r|j|qHt||rH|j|qHWq>Wx|D]}|j|qWx|D]}|j|qWx&|D]}t|tr|}|j|qW|S)NHTTPSConnection)rrr.r)rrr+r*r/r-rrclientr]rEsetrr issubclassaddremover)rrKZdefault_classesskipklassZcheckhrLrLrMr2@s0             c@s(eZdZdZddZddZddZdS) ricCs ||_dS)N)parent)rrrLrLrMrgszBaseHandler.add_parentcCsdS)NrL)rrLrLrMrjszBaseHandler.closecCst|dsdS|j|jkS)N handler_orderT)rr)rotherrLrLrM__lt__ns zBaseHandler.__lt__N)rrrrrrrrLrLrLrMrdsc@seZdZdZddZeZdS)r/icCsJ|j|j|j}}}d|ko*dknsF|jjd|||||}|S)Ni,r)codemsgrVrr)rrurrrrrLrLrM http_response{s z HTTPErrorProcessor.http_responseN)rrrrrhttps_responserLrLrLrMr/ws c@seZdZddZdS)rcCst|j||||dS)N)rrp)rrrdrrrrLrLrMrsz*HTTPDefaultErrorHandler.http_error_defaultN)rrrrrLrLrLrMrsc@s4eZdZdZdZddZddZeZZZ dZ dS) r c sx|j}|dkr|dkp&|dko&|dks:t|j|||||jdd }dtfd d |jjD}t|||jddS)N-./3rHEADr z%20content-length content-typec3s&|]\}}|jkr||fVqdS)N)rt).0kv)CONTENT_HEADERSrLrM sz7HTTPRedirectHandler.redirect_request..T)rer}r~)rrrr)rr)rrr)rr) rrrpreplacerrer{rr}) rrrdrrrenewurlmZ newheadersrL)rrMrs  z$HTTPRedirectHandler.redirect_requestc CsNd|kr|d}nd|kr$|d}ndSt|}|jdkrRt||d||f|||j rp|jrpt|}d|d <t|}t|d tj d }t |j |}|j ||||||}|dkrdSt |d r|j} |_| j|d |jkst| |jkrt|j ||j|||ni} |_|_| j|d d| |<|j|j|jj||jdS)Nlocationurirrftprnz+%s - Redirection to url '%s' is not allowed/r:z iso-8859-1)encodingsafe redirect_dictrrR)rJ)rrrrn)rschemerrXZnetlocrrr stringZ punctuationrrprrrr max_repeatsr`max_redirectionsinf_msgr_rrrGrJ) rrrdrrrerurlpartsnewZvisitedrLrLrMhttp_error_302s@       z"HTTPRedirectHandler.http_error_302zoThe HTTP server returned a redirect error that would lead to an infinite loop. The last 30x error message was: N) rrrrrrr http_error_301http_error_303http_error_307rrLrLrLrMrs &< c Cst|\}}|jds d}|}n:|jds6td||jdd}|dkrNd}|d|}t|\}}|dk r|t|\}}nd}}||||fS)Nrz//zproxy URL with no authority: %rr:rRrS)r rrBrrr) proxyrZr_scheme authorityendZuserinfohostportuserpasswordrLrLrM _parse_proxys       rc@s"eZdZdZdddZddZdS)rdNcCsL|dkrt}||_x2|jD]&\}}t|d||||jfddqWdS)Nz%s_opencSs ||||S)NrL)rr rrrLrLrM%sz'ProxyHandler.__init__..)r5proxiesr{setattrr)rrrrHrLrLrMrs  zProxyHandler.__init__c Cs|j}t|\}}}}|dkr"|}|jr6t|jr6dS|rv|rvdt|t|f} tj| jjd} |j dd| t|}|j ||||ks|dkrdS|j j ||j dSdS)Nz%s:%sasciizProxy-authorizationzBasic r)rJ)rrrv proxy_bypassr base64 b64encodeencodedecoder|rrrGrJ) rrr rZ orig_typeZ proxy_typerrrZ user_passZcredsrLrLrMr(s   zProxyHandler.proxy_open)N)rrrrrrrLrLrLrMrs c@s6eZdZddZddZddZd dd Zd d Zd S)r cCs i|_dS)N)passwd)rrLrLrMrFszHTTPPasswordMgr.__init__cs`t|tr|g}|jkr$ij|<x6dD].tfdd|D}||fj||<q*WdS)NTFcsg|]}j|qSrL) reduce_uri)ru) default_portrrLrM Qsz0HTTPPasswordMgr.add_password..)TF)rrrtuple)rrealmrrr reduced_urirL)r"rrM add_passwordIs    zHTTPPasswordMgr.add_passwordc Cs`|jj|i}xLdD]D}|j||}x2|jD]&\}}x|D]}|j||r<|SqdD]6}|j||}x$|jD]}|j||r|j|SqWqWdS)NTF)TF)r r4r()rr)r"r*rrLrLrMr9s     z-HTTPPasswordMgrWithPriorAuth.is_authenticated)F)F)rrrrr'r8r9 __classcell__rLrL)r7rMr"s  c@sTeZdZejdejZdddZddZddZ d d Z d d Z d dZ e Z e ZdS)r#z1(?:^|,)[ ]*([^ ,]+)[ ]+realm=(["']?)([^"']*)\2NcCs"|dkrt}||_|jj|_dS)N)r rr')rZ password_mgrrLrLrMrsz!AbstractBasicAuthHandler.__init__ccstd}xFtjj|D]6}|j\}}}|d kr:tjdtd||fVd}qW|sp|rb|jd}nd}|dfVdS) NF"'zBasic Auth Realm was unquotedTrrn)r;r<)r#rxfinditergroupsr?r@ UserWarningsplit)rheaderZfound_challengeZmorr r%rLrLrM _parse_realms z%AbstractBasicAuthHandler._parse_realmc Cs~|j|}|sdSd}xL|D]D}x>|j|D]0\}}|jdkrF|}q,|dk r,|j|||Sq,WqW|dk rztd|fdS)NbasiczBAbstractBasicAuthHandler does not support the following scheme: %r)Zget_allrDrtretry_http_basic_authrB) rauthreqrvrreZ unsupportedrCrr%rLrLrMhttp_error_auth_reqeds   z.AbstractBasicAuthHandler.http_error_auth_reqedcCs||jj||\}}|dk rtd||f}dtj|jjd}|j|jd|krTdS|j|j||j j ||j dSdSdS)Nz%s:%szBasic r)rJ) rr+rrrrrq auth_headerrrrGrJ)rrvrr%rpwrawauthrLrLrMrFs z.AbstractBasicAuthHandler.retry_http_basic_authcCsxt|jd s|jj|j r"|S|jdst|jjd|j\}}dj||j}tj |j }|j ddj|j |S)Nr9 Authorizationz{0}:{1}zBasic {}) rrr9rprr+rrrZstandard_b64encoderrstrip)rrrrZ credentialsZauth_strrLrLrM http_requests z%AbstractBasicAuthHandler.http_requestcCsLt|jdrHd|jko dknr8|jj|jdn|jj|jd|S)Nr9ri,TF)rrrr8rp)rrrrLrLrMrs  z&AbstractBasicAuthHandler.http_response)N)rrrrecompileIr>rrDrHrFrOr https_requestrrLrLrLrMr#s   c@seZdZdZddZdS)r$rMcCs|j}|jd|||}|S)Nzwww-authenticate)rprH)rrrdrrrerHrrLrLrMhttp_error_401 s z#HTTPBasicAuthHandler.http_error_401N)rrrrIrTrLrLrLrMr$sc@seZdZdZddZdS)r%zProxy-authorizationcCs|j}|jd|||}|S)Nzproxy-authenticate)rvrH)rrrdrrrerrrLrLrMhttp_error_407+s z$ProxyBasicAuthHandler.http_error_407N)rrrrIrUrLrLrLrMr%'sc@sNeZdZdddZddZddZdd Zd d Zd d ZddZ ddZ dS)r&NcCs4|dkrt}||_|jj|_d|_d|_d|_dS)Nr)r rr'retried nonce_count last_nonce)rrrLrLrMrEs z"AbstractDigestAuthHandler.__init__cCs d|_dS)Nr)rV)rrLrLrMreset_retry_countNsz+AbstractDigestAuthHandler.reset_retry_countcCs||j|d}|jdkr*t|jdd|dn|jd7_|rx|jd}|jdkr`|j||S|jdkrxtd|dS) Nizdigest auth failedrRrZdigestrEzEAbstractDigestAuthHandler does not support the following scheme: '%s')rrVrrprBrtretry_http_digest_authrB)rrIrvrrerGrrLrLrMrHQs        z/AbstractDigestAuthHandler.http_error_auth_reqedcCsz|jdd\}}ttdt|}|j||}|rvd|}|jj|jd|krRdS|j|j||j j ||j d}|SdS)NrrRz Digest %s)rJ) rBparse_keqv_listfilterparse_http_listget_authorizationrerrIrrrGrJ)rrrLtokenZ challengechalZauth_valZresprLrLrMr[es z0AbstractDigestAuthHandler.retry_http_digest_authcCs@d|j|tjf}|jdtd}tj|j}|ddS)Nz %s:%s:%s:rrQ)rWtimeZctimer _randombyteshashlibsha1 hexdigest)rnoncesbdigrLrLrM get_cnonceqsz$AbstractDigestAuthHandler.get_cnoncecCsy6|d}|d}|jd}|jdd}|jdd}Wntk rJdSX|j|\}} |dkrfdS|jj||j\} } | dkrdS|jdk r|j|j|} nd} d| || f} d|j|j f}|d kr.||j kr|j d 7_ n d |_ ||_ d |j }|j |}d ||||||f}| || |}n2|dkrT| || d|||f}n t d |d| |||j |f}|r|d|7}| r|d| 7}|d|7}|r|d||f7}|S)Nr%rhqop algorithmMD5opaquez%s:%s:%sz%s:%srLrRz%08xz%s:%s:%s:%s:%szqop '%s' is not supported.z>username="%s", realm="%s", nonce="%s", uri="%s", response="%s"z , opaque="%s"z , digest="%s"z, algorithm="%s"z, qop=auth, nc=%s, cnonce="%s")rKeyErrorget_algorithm_implsrr+rprIget_entity_digestrrrXrWrlr)rrrar%rhrmrnrpHKDrrJZentdigZA1ZA2ZncvalueZcnonceZnoncebitZrespdigr1rLrLrMr_|sV              z+AbstractDigestAuthHandler.get_authorizationcsD|dkrddn|dkr$ddn td|fdd}|fS)NrocSstj|jdjS)Nr)reZmd5rrg)xrLrLrMrsz?AbstractDigestAuthHandler.get_algorithm_impls..ZSHAcSstj|jdjS)Nr)rerfrrg)rvrLrLrMrsz.Unsupported digest authentication algorithm %rcsd||fS)Nz%s:%srL)rid)rtrLrMrs)rB)rrnrurL)rtrMrrs   z-AbstractDigestAuthHandler.get_algorithm_implscCsdS)NrL)rrIrarLrLrMrssz+AbstractDigestAuthHandler.get_entity_digest)N) rrrrrYrHr[rlr_rrrsrLrLrLrMr&:s   < c@seZdZdZdZddZdS)r'rMicCs*t|jd}|jd|||}|j|S)NrRzwww-authenticate)rrprHrY)rrrdrrrervretryrLrLrMrTs  z$HTTPDigestAuthHandler.http_error_401N)rrrrIrrTrLrLrLrMr'sc@seZdZdZdZddZdS)r(zProxy-AuthorizationicCs"|j}|jd|||}|j|S)Nzproxy-authenticate)rvrHrY)rrrdrrrervrxrLrLrMrUs  z%ProxyDigestAuthHandler.http_error_407N)rrrrIrrUrLrLrLrMr(sc@s6eZdZd ddZddZddZdd Zd d Zd S)AbstractHTTPHandlerrcCs ||_dS)N) _debuglevel)r debuglevelrLrLrMrszAbstractHTTPHandler.__init__cCs ||_dS)N)rz)rlevelrLrLrMset_http_debuglevelsz'AbstractHTTPHandler.set_http_debuglevelcCstjjj|j|jS)N)rrHTTPConnection_get_content_lengthrIr)rrurLrLrMrsz'AbstractHTTPHandler._get_content_lengthc Cs |j}|std|jdk r|j}t|tr8d}t||jdsN|jdd|jd r|jd r|j|}|dk r|jdt|n |jdd|}|j rt |j \}}t |\}} |jds|jd|x2|j jD]&\} } | j} |j| s|j| | qW|S) Nz no host givenz\POST data should be bytes, an iterable of bytes, or a file object. It cannot be of type str.z Content-typez!application/x-www-form-urlencodedzContent-lengthzTransfer-encodingZchunkedro)rvrrIrrrrrrrr rr rrr) rrurvrIrZcontent_lengthZsel_hostrZselZsel_pathr[rrLrLrM do_request_s>          zAbstractHTTPHandler.do_request_c  s\|j}|std||fd|ji|}|j|jt|jjtfdd|jj Ddd<tddj D|j ri}d}|kr|||<|=|j |j |d y`y&|j |j |j|j|jd d Wn,tk r }zt|WYdd}~XnX|j} Wn|jYnX|jrF|jjd|_|j| _| j| _| S) Nz no host givenrJc3s"|]\}}|kr||fVqdS)NrL)rrr)rerLrMr)sz.AbstractHTTPHandler.do_open..r Connectioncss|]\}}|j|fVqdS)N)title)rr[rrLrLrMr6szProxy-Authorization)rezTransfer-encoding)Zencode_chunked)rvrrJZset_debuglevelrzrrxrrer{rzZ set_tunnelrurrrIrrm getresponserZsockrrHreasonr) rZ http_classrZhttp_conn_argsrvrZtunnel_headersZproxy_auth_hdrerrrrL)rerMrs@  "     zAbstractHTTPHandler.do_openN)r)rrrrr}rrrrLrLrLrMrys  &ryc@seZdZddZejZdS)r)cCs|jtjj|S)N)rrrr~)rrrLrLrM http_open`szHTTPHandler.http_openN)rrrrryrrOrLrLrLrMr)^src@s$eZdZdddZddZejZdS)rErNcCstj||||_||_dS)N)ryr_context_check_hostname)rr{r>check_hostnamerLrLrMris zHTTPSHandler.__init__cCs|jtjj||j|jdS)N)r>r)rrrrrr)rrrLrLrM https_openns zHTTPSHandler.https_open)rNN)rrrrrryrrSrLrLrLrMrEgs rEc@s.eZdZdddZddZddZeZeZdS) rNcCs$ddl}|dkr|jj}||_dS)Nr)Zhttp.cookiejar cookiejarZ CookieJar)rrrrLrLrMrws zHTTPCookieProcessor.__init__cCs|jj||S)N)rZadd_cookie_header)rrurLrLrMrO}s z HTTPCookieProcessor.http_requestcCs|jj|||S)N)rZextract_cookies)rrurrLrLrMrsz!HTTPCookieProcessor.http_response)N)rrrrrOrrSrrLrLrLrMrvs  c@seZdZddZdS)r.cCs|j}td|dS)Nzunknown url type: %s)rr)rrrrLrLrMrszUnknownHandler.unknown_openN)rrrrrLrLrLrMr.scCsRi}xH|D]@}|jdd\}}|ddkrB|ddkrB|dd}|||<q W|S)N=rRrr;rSrS)rB)lZparsedZeltrrrLrLrMr\s   r\cCsg}d}d}}xt|D]l}|r,||7}d}q|rV|dkr@d}qn |dkrLd}||7}q|dkrn|j|d}q|dkrzd}||7}qW|r|j|dd|DS) NrnF\Tr;,cSsg|] }|jqSrL)rN)rpartrLrLrMr#sz#parse_http_list..)r])riresrescaper ZcurrLrLrMr^s4     r^c@s(eZdZddZdZddZddZdS)r*cCs\|j}|dddkrN|dddkrN|jrN|jdkrN|j|jkrXtdn |j|SdS)Nr:z//r=r localhostz-file:// scheme is supported only on localhost)rrv get_namesropen_local_file)rrrHrLrLrM file_opens &  zFileHandler.file_openNc Cs`tjdkrZy*ttjddtjtjdt_Wn$tjk rXtjdft_YnXtjS)Nrr:)r*namesr$rgethostbyname_ex gethostnamegaierror gethostbyname)rrLrLrMrs  zFileHandler.get_namescCsddl}ddl}|j}|j}t|}ytj|}|j}|jj |j dd} |j |d} |j d| pbd|| f} |r~t |\}} | s| rt||jkr|rd||} nd|} tt|d| | SWn*tk r}zt|WYdd}~XnXtddS) NrT)usegmtz6Content-type: %s Content-length: %d Last-modified: %s z text/plainzfile://rbzfile not on local host) email.utils mimetypesrvrr4rWstatst_sizeutils formatdatest_mtime guess_typemessage_from_stringr _safe_gethostbynamerrrGrmr)rremailrrvrbZ localfilestatsrimodifiedmtyperer.ZorigurlexprLrLrMrs0  zFileHandler.open_local_file)rrrrrrrrLrLrLrMr*s  c Cs&y tj|Stjk r dSXdS)N)rrr)rvrLrLrMrs rc@seZdZddZddZdS)r+cCs.ddl}ddl}|j}|s"tdt|\}}|dkr>|j}nt|}t|\}}|rdt|\}}nd}t |}|pvd}|p~d}yt j |}Wn*t k r}zt|WYdd}~XnXt |j\} } | jd} ttt | } | dd| d} } | r| d r| dd} y|j||||| |j} | r8dp:d}x:| D]2}t|\}}|jdkrB|dkrB|j}qBW| j| |\}}d}|j|jd}|r|d |7}|dk r|dkr|d|7}tj|}t|||jS|jk r(}z$td|}|jtj dWYdd}~XnXdS)Nrzftp error: no host givenrnrrRrRDraArrwzContent-type: %s zContent-length: %d z ftp error: %rr:rSrS)rrrrRrwr)!ftplibrrvrr FTP_PORTr^rrr rrrmrrrBrmap connect_ftprJrrtupperretrfilerrprrr all_errorswith_tracebacksysexc_info)rrrrrvr.rrrrXattrsdirsrNfwrattrrrdretrlenrerrexcrLrLrMftp_opens\           zFTPHandler.ftp_openc Cst||||||ddS)NF) persistent) ftpwrapper)rrrrvr.rrJrLrLrMr1szFTPHandler.connect_ftpN)rrrrrrLrLrLrMr+s5c@s<eZdZddZddZddZddZd d Zd d Zd S)r,cCs"i|_i|_d|_d|_d|_dS)Nr<rb)cacherJsoonestdelay max_conns)rrLrLrMr8s zCacheFTPHandler.__init__cCs ||_dS)N)r)rtrLrLrM setTimeout?szCacheFTPHandler.setTimeoutcCs ||_dS)N)r)rrrLrLrM setMaxConnsBszCacheFTPHandler.setMaxConnscCsr|||dj||f}||jkr4tj|j|j|<n,t|||||||j|<tj|j|j|<|j|j|S)Nr)joinrrcrrJr check_cache)rrrrvr.rrJrrLrLrMrEs  zCacheFTPHandler.connect_ftpcCstj}|j|krTx@t|jjD].\}}||kr"|j|j|j|=|j|=q"Wtt|jj|_t |j|j krx6t|jjD]$\}}||jkr|j|=|j|=PqWtt|jj|_dS)N) rcrrrJr{rrminvaluesr`r)rrrrrLrLrMrPs   zCacheFTPHandler.check_cachecCs4x|jjD] }|jq W|jj|jjdS)N)rrrclearrJ)rconnrLrLrM clear_cacheds  zCacheFTPHandler.clear_cacheN) rrrrrrrrrrLrLrLrMr,5s  c@seZdZddZdS)r-cCs~|j}|jdd\}}|jdd\}}t|}|jdrNtj|}|dd}|sVd}tjd|t|f}t t j |||S) N:rRrz;base64ztext/plain;charset=US-ASCIIz$Content-type: %s Content-length: %d i) rprBrendswithr decodebytesrrr`rioBytesIO)rrrHrrIZ mediatypererLrLrM data_openks    zDataHandler.data_openN)rrrrrLrLrLrMr-jsrnt)r4r3cCst|S)N)r )pathnamerLrLrMr4scCst|S)N)r )rrLrLrMr3sc@seZdZdZdeZd)ddZddZddZd d Z d d Z d*d dZ d+ddZ d,ddZ d-ddZddZd.ddZd/ddZddZerddZd0dd Zd!d"Zd#d$Zd%d&Zd1d'd(ZdS)2r8NzPython-urllib/%scKszdd|jji}tj|tdd|dkr.t}||_|jd|_|jd|_ d|j fd g|_ g|_ t j|_d|_t|_dS) NzW%(class)s style of invoking requests is deprecated. Use newer urlopen functions/methodsclassr=) stacklevelkey_file cert_filez User-AgentAccept*/*)rr)r7rr?r@rAr5rrrrversionr_URLopener__tempfilesrWrl_URLopener__unlink tempcacheftpcache)rrZx509rrLrLrMrs  zURLopener.__init__cCs |jdS)N)r)rrLrLrM__del__szURLopener.__del__cCs |jdS)N)cleanup)rrLrLrMrszURLopener.closec CsZ|jrFx2|jD](}y|j|Wqtk r4YqXqW|jdd=|jrV|jjdS)N)rrrmrr)rrNrLrLrMrs   zURLopener.cleanupcGs|jj|dS)N)rr])rrrLrLrM addheaderszURLopener.addheadercCsntt|}t|dd}|jrL||jkrL|j|\}}t|d}t|||St|\}}|s`d}||jkr|j|}t|\}} t| \} } | |f}nd}d|} ||_ | j dd} t ||  s| dkr|r|j |||S|j ||Sy,|dkrt|| |St|| ||SWnVttfk r.Yn<tk rh} ztd | jtjd WYdd} ~ XnXdS) Nz%/:=&?~#+!$,;'@()*[]|)rrrNZopen_-rrz socket errorr:)rrr rrGrr rr rrropen_unknown_proxy open_unknownrrrrmrrr)rrrIrbrerdurltyperHr  proxyhostrvrr[rrLrLrMrGs<             zURLopener.opencCst|\}}tdd|dS)Nz url errorzunknown url type)r rm)rrrIrrHrLrLrMrs zURLopener.open_unknowncCs t|\}}tdd||dS)Nz url errorzinvalid proxy for %s)r rm)rr rrIrrHrLrLrMr s zURLopener.open_unknown_proxyc Cs&tt|}|jr&||jkr&|j|St|\}}|dkr| sH|dkry.|j|}|j}|jtt|d|fSt k r} zWYdd} ~ XnX|j ||}zH|j} |rt |d} n|ddl } t|\} }t|pd\} }t |pd\}} t |pd\}} tjj|d}| j|\}}|jj|tj|d} z|| f}|jdk r^||j|<d }d }d}d}d| krt| d }|r||||xH|j|}|sP|t|7}| j||d7}|r||||qWWd| jXWd|jX|dkr"||kr"td ||f||S) NrNrRrOrrnirQzcontent-lengthzContent-Lengthz1retrieval incomplete: got only %i out of %i bytesi rS)rrrr rrVrr4r rmrGrZrrrWrXsplitextZmkstemprr]fdopenr^r_r`rar)rrHrbrcrIrZurl1rdrrrerfrZZgarbagerXsuffixfdrgrhrir_rjrkrLrLrMretrievesl                  zURLopener.retrievecCs(d}d}t|trtd |tt|d | | Std dS)NrT)rz6Content-Type: %s Content-Length: %d Last-modified: %s z text/plainrRrzfile://rr:z./zAlocal file url may start with / or file:. Unknown url of type: %sz#local file error: not on local host)rrr r4rWrrmrstrerrorrbrrrrrrrrGr rrrthishostrB)rrHrrrvrNZ localnamererirrreZurlfiler.rLrLrMrs:     zURLopener.open_local_filecCst|tstdddl}t|\}}|s2tdt|\}}t|\}}|r\t|\}}nd}t|}t|ppd}t|p|d}t j |}|sddl }|j }nt |}t|\}} t|}|jd} | dd| d} } | o| d r| dd} | r| d rd| d<|||dj| f} t|jtkrlx8t|jD]*} | | kr>|j| }|j| =|jq>Wy| |jkrt||||| |j| <| sd}nd}x:| D]2}t|\}}|jd kr|dkr|j}qW|j| j| |\}}|jd|d}d}|r|d|7}|dk r:|dkr:|d|7}tj|}t||d|Stk r}z td|j t!j"dWYdd}~XnXdS)NzCftp error: proxy support for ftp protocol currently not implementedrzftp error: no host givenrnrrRrrRrrrrrwzftp:zContent-Type: %s zContent-Length: %d z ftp error %rr:rSrS)rrrrRrwr)#rrrrr r rrr rrrrr^rrBrr`r MAXFTPCACHErrrrrtrrrrrr ftperrorsrrr)rrHrrvrXr.rrrrrrNrrrrrrrdrrrerrLrLrMopen_ftpsp                   zURLopener.open_ftpc Cs<t|tstdy|jdd\}}Wntk rDtddYnX|sNd}|jd}|dkrd ||dkr||dd}|d|}nd }g}|jd tj d tj tj|jd ||dkrt j |j djd}nt|}|jdt||jd |j|dj|}tj|}tj|}t|||S)NzEdata error: proxy support for data protocol currently not implementedrrRz data errorz bad data URLztext/plain;charset=US-ASCII;rrrnzDate: %sz%a, %d %b %Y %H:%M:%S GMTzContent-type: %srrzlatin-1zContent-Length: %d )rrrrBrBrmrfindr]rcZstrftimeZgmtimerrrrr r`rrrrStringIOr) rrHrIrZsemirrrefrLrLrM open_data3s6        zURLopener.open_data)N)N)N)N)NNN)N)N)N)N)rrrrrrrrrrrrGrrrrrrrrCrrrrrrrLrLrLrMr8s,  $   B\     :c@seZdZddZddZd"ddZdd Zd#d d Zd$d d Zd%ddZ d&ddZ d'ddZ d(ddZ d)ddZ d*ddZd+ddZd,ddZd d!ZdS)-r9cOs(tj|f||i|_d|_d|_dS)Nrr)r8r auth_cachetriesmaxtries)rrr6rLrLrMr`szFancyURLopener.__init__cCst||d||S)Nzhttp:)r)rrHrdrrrerLrLrMrfsz!FancyURLopener.http_error_defaultNc Csn|jd7_zR|jrJ|j|jkrJt|dr4|j}n|j}|||dd|S|j||||||}|Sd|_XdS)NrRhttp_error_500iz)Internal Server Error: Redirect Recursionr)r r rr rredirect_internal) rrHrdrrrerIrrgrLrLrMr js  zFancyURLopener.http_error_302c Csxd|kr|d}nd|kr$|d}ndS|jt|jd||}t|}|jd krnt|||d||||j|S) Nrrrrrrrnz( Redirection to url '%s' is not allowed.)rrrrn)rrrrrrrG) rrHrdrrrerIrrrLrLrMr |s   z FancyURLopener.redirect_internalcCs|j||||||S)N)r )rrHrdrrrerIrLrLrMr szFancyURLopener.http_error_301cCs|j||||||S)N)r )rrHrdrrrerIrLrLrMr szFancyURLopener.http_error_303cCs2|dkr|j||||||S|j|||||SdS)N)r r)rrHrdrrrerIrLrLrMr szFancyURLopener.http_error_307Fc Csd|krtj|||||||d}tjd|} | sHtj||||||| j\} } | jdkrttj|||||||stj||||||d|jd} |dkrt|| || St|| || |SdS)Nzwww-authenticatez![ ]*([^ ]+)[ ]+realm="([^"]*)"rEZretry_ _basic_auth)r8rrPmatchr@rtrr) rrHrdrrrerIrxstuffrrr%r[rLrLrMrTs&         zFancyURLopener.http_error_401c Csd|krtj|||||||d}tjd|} | sHtj||||||| j\} } | jdkrttj|||||||stj||||||d|jd} |dkrt|| || St|| || |SdS)Nzproxy-authenticatez![ ]*([^ ]+)[ ]+realm="([^"]*)"rEZ retry_proxy_r )r8rrPrr@rtrr) rrHrdrrrerIrxrrrr%r[rLrLrMrUs&         zFancyURLopener.http_error_407cCst|\}}d||}|jd}t|\}} t| \} } | jdd} | | d} |j| || \} } | pl| srdSdt| ddt| dd| f} d| | |jd<|dkr|j|S|j||SdS)Nzhttp://r@rRz%s:%s@%srn)r)r rr rget_user_passwdr rG)rrHr%rIrvrrr rr proxyselectorrrrrLrLrMretry_proxy_http_basic_auths         z*FancyURLopener.retry_proxy_http_basic_authcCst|\}}d||}|jd}t|\}} t| \} } | jdd} | | d} |j| || \} } | pl| srdSdt| ddt| dd| f} d| | |jd<|dkr|j|S|j||SdS)Nzhttps://rrrRz%s:%s@%srn)r)r rr rrr rG)rrHr%rIrvrrr rrrrrrrLrLrMretry_proxy_https_basic_auths         z+FancyURLopener.retry_proxy_https_basic_authc Cst|\}}|jdd}||d}|j|||\}}|p>|sDdSdt|ddt|dd|f}d||} |dkr|j| S|j| |SdS)NrrRz%s:%s@%srn)rzhttp://)r rrr rG) rrHr%rIrvrrrrrrLrLrMrFs     z$FancyURLopener.retry_http_basic_authc Cst|\}}|jdd}||d}|j|||\}}|p>|sDdSdt|ddt|dd|f}d||} |dkr|j| S|j| |SdS)NrrRz%s:%s@%srn)rzhttps://)r rrr rG) rrHr%rIrvrrrrrrLrLrMretry_https_basic_auth s     z%FancyURLopener.retry_https_basic_authrcCs`|d|j}||jkr2|r(|j|=n |j|S|j||\}}|sJ|rX||f|j|<||fS)Nr)rtrprompt_user_passwd)rrvr%rrrrrLrLrMr s   zFancyURLopener.get_user_passwdc CsTddl}y,td||f}|jd|||f}||fStk rNtdSXdS)NrzEnter username for %s at %s: z#Enter password for %s in %s at %s: )NN)getpassinputKeyboardInterruptprint)rrvr%rrrrLrLrMr$ sz!FancyURLopener.prompt_user_passwd)N)N)N)N)NF)NF)N)N)N)N)r)rrrrrr r r r r rTrUrrrFrrrrLrLrLrMr9]s"           cCstdkrtjdatS)Nr) _localhostrrrLrLrLrMr4 s rc CsPtdkrLyttjtjdaWn(tjk rJttjddaYnXtS)Nr:r) _thishostr$rrrrrLrLrLrMr< s rcCstdkrddl}|jatS)Nr) _ftperrorsrr)rrLrLrMrG srcCstdkrtjdatS)Nrn) _noheadersrrrLrLrLrM noheadersP s rc@sFeZdZdddZddZddZd d Zd d Zd dZddZ dS)rNTc CsX||_||_||_||_||_||_d|_||_y |jWn|j YnXdS)Nr) rrrvr.rrJrefcount keepaliveinitr)rrrrvr.rrJrrLrLrMr] s zftpwrapper.__init__cCs\ddl}d|_|j|_|jj|j|j|j|jj|j |j dj |j }|jj |dS)Nrr)rbusyZFTPrZconnectrvr.rJZloginrrrrcwd)rrZ_targetrLrLrMr"m s  zftpwrapper.initc -Csddl}|j|dkr"d}d}n d|}d}y|jj|Wn*|jk rh|j|jj|YnXd}|r| ryd|}|jj|\}}WnR|jk r}z4t|ddd krt d |j t j d WYdd}~XnX|s|jjd|rn|jj } zJy|jj|Wn4|jk rP}zt d ||WYdd}~XnXWd|jj| Xd |}nd }|jj|\}}d|_t|jd|j} |jd7_|j| |fS)NrrwrzTYPE ArRzTYPE zRETR r=Z550z ftp error: %rr:zLIST ZLISTr)rwr)r endtransferrZvoidcmdrr"Z ntransfercmdZ error_permrrrrrpwdr$r#rmakefile file_closer r) rrNrrcmdisdirrrrr&ZftpobjrLrLrMrv sN     $ zftpwrapper.retrfilecCs d|_dS)Nr)r#)rrLrLrMr% szftpwrapper.endtransfercCsd|_|jdkr|jdS)NFr)r!r  real_close)rrLrLrMr s zftpwrapper.closecCs4|j|jd8_|jdkr0|j r0|jdS)NrRr)r%r r!r+)rrLrLrMr( szftpwrapper.file_closec Cs2|jy|jjWntk r,YnXdS)N)r%rrr)rrLrLrMr+ s zftpwrapper.real_close)NT) rrrrr"rr%rr(r+rLrLrLrMrZ s  -rcCsi}xBtjjD]4\}}|j}|r|dddkr|||dd<qWdtjkr^|jddxXtjjD]J\}}|dddkrj|j}|r|||dd<qj|j|dd dqjW|S) N_proxyZREQUEST_METHODriiiii)rWenvironr{rtr)rr[rrLrLrMgetproxies_environment s   r/c Cs|dkrt}y |d}Wntk r.dSX|dkr.r.z (.+\.)?%s$) r/rqr rBlstriprPrrrR)rvrZno_proxyhostonlyr.Z no_proxy_listr[patternrLrLrMproxy_bypass_environment s&     r6c Csddlm}t|\}}dd}d|kr4|dr4dSd}x|jdfD]}|sPqFtjd |}|dk r|dkrytj|}||}Wntk rwFYnX||jd } |jd } | dkrd |jd j dd } nt | d d} d | } || ?| | ?krdSqF|||rFdSqFWdS)Nr)fnmatchcSsh|jd}ttt|}t|dkr<|ddddgdd}|dd>|dd>B|dd>B|d BS) Nr2rrrRrbr:rQr=)rBrrr^r`)ZipAddrr-rLrLrMip2num s   z,_proxy_bypass_macosx_sysconf..ip2numr2Zexclude_simpleT exceptionsz(\d+(?:\.\d+)*)(/\d+)?rRr:rQ F) r7r rrPrrrrmgroupcountr^) rvproxy_settingsr7r4r.r9ZhostIPrrr1maskrLrLrM_proxy_bypass_macosx_sysconf s:        r@darwin)_get_proxy_settings _get_proxiescCst}t||S)N)rBr@)rvr>rLrLrMproxy_bypass_macosx_sysconf: srDcCstS)N)rCrLrLrLrMgetproxies_macosx_sysconf> srEcCs t}|rt||St|SdS)N)r/r6rD)rvrrLrLrMrH s rcCs tp tS)N)r/rErLrLrLrMr5U scCsi}y ddl}Wntk r$|SXy|j|jd}|j|dd}|rt|j|dd}d|krx|jdD]4}|jdd\}}tjd|sd ||f}|||<qrWn>|dd d kr||d <n$d ||d <d||d<d||d<|j Wnt t t fk rYnX|S)Nrz;Software\Microsoft\Windows\CurrentVersion\Internet Settings ProxyEnableZ ProxyServerrrrRz ^([^/:]+)://z%s://%srZzhttp:rz http://%sz https://%srzftp://%sr) winreg ImportErrorOpenKeyHKEY_CURRENT_USER QueryValueExrrBrPrZClosermrBr)rrGinternetSettings proxyEnableZ proxyServerprZaddressrLrLrMgetproxies_registryZ s8          rOcCs tp tS)N)r/rOrLrLrLrMr5 sc &Cs~y ddl}Wntk r dSXy6|j|jd}|j|dd}t|j|dd}Wntk rldSX| sz| r~dSt|\}}|g}y tj |}||kr|j |Wntk rYnXy tj |}||kr|j |Wntk rYnX|j d}xp|D]h} | dkr*d|kr*dS| j dd } | j d d } | j d d} x$|D]} tj| | tjrTdSqTWqWdS) Nrz;Software\Microsoft\Windows\CurrentVersion\Internet SettingsrFZ ProxyOverriderzr2rRz\.r1z.*?)rGrHrIrJrKrrmr rrr]ZgetfqdnrBrrPrrR) rvrGrLrMZ proxyOverrideZrawHostr.ZaddrZfqdnr2rrLrLrMproxy_bypass_registry sR              rQcCs t}|rt||St|SdS)N)r/r6rQ)rvrrLrLrMr s )NNN)N)~rrrreZ http.clientrrrWr/rPrrrrc collectionsrZrTr?Z urllib.errorrrrZ urllib.parserrrrr r r r r rrrrrrrrrZurllib.responserrrDrHrC__all__ version_inforrFrr0r1r\r6r7rQASCIIrrrwrrr2rr/rrrrr r!r"r#r$r%urandomrdr&r'r(ryr)rrrEr]rr.r\r^r*rr+r,r-rr[Z nturl2pathr4r3rr8r9rrrrrrrrrr/r6r@platformZ_scproxyrBrCrDrErr5rOrQrLrLrLrMTsP   T ?n$q*@ o  v  +3:5! AW  _ #<    - 2