B
    9aГ                 @   s  d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ dd	lmZ ddlmZ  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZD eBr0ddlmEZE ddlFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZP ddlmQZQ ddlRmSZS ddlTmUZU ddl!mVZV eGeWeWeHeW f ZXeGeIdf ZYeGeZeQeYeHeZ f Z[d d!gZ\d"d#d$d%d&d'gZ]e^e_Z`d(d) Zad*d+ ZbG d,d- d-ecZdd.d/ ZeG d0d1 d1ecZfd2d3 Zgd4d5 ZhdJd6d7ZidKd8d9ZjG d:d! d!ekZld;d< Zmd=d> Znd?d@ ZodAdB Zpe	qdCe	jrZsdDdE ZtG dFdG dGekZuedHdIZvdS )Lz!Routines related to PyPI, indexes    )absolute_importN)
namedtuple)html5librequestssix)unescape)
specifiers)canonicalize_name)parse)
RetryErrorSSLError)request)HAS_TLSis_urlpath_to_urlurl_to_path)BestVersionAlreadyInstalledDistributionNotFoundInvalidWheelFilenameUnsupportedWheel)InstallationCandidate)FormatControl)PyPI)Link)get_supported)	ipaddress)
indent_log)ARCHIVE_EXTENSIONSSUPPORTED_EXTENSIONSWHEEL_EXTENSIONnormalize_pathredact_password_from_url)check_requires_python)MYPY_CHECK_RUNNING)Wheel)Logger)
TupleOptionalAnyListUnionCallableSetSequenceIterableMutableMapping)_BaseVersion)Response)InstallRequirement)
PipSession.r   PackageFinder)https*r6   )r6   	localhostr6   )r6   z127.0.0.0/8r6   )r6   z::1/128r6   )filer6   N)sshr6   r6   c             C   sB   ddl m} x0|jD ]&}|  |r| t| dkr|S qW dS )zgLook for VCS schemes in the URL.

    Returns the matched VCS scheme, or None if there's no match.
    r   )
VcsSupportz+:N)pip._internal.vcsr:   schemeslower
startswithlen)urlr:   scheme rB   4/tmp/pip-install-u9kq1d1e/pip/pip/_internal/index.py_match_vcs_schemeM   s
    rD   c             C   s*   t | j}xtD ]}||rdS qW dS )z2Return whether the URL looks like an archive.
    TF)r   filenamer   endswith)r@   rE   bad_extrB   rB   rC   _is_url_like_archiveZ   s
    


rH   c                   s   e Zd Z fddZ  ZS )_NotHTMLc                s"   t t| || || _|| _d S )N)superrI   __init__content_typerequest_desc)selfrL   rM   )	__class__rB   rC   rK   f   s    z_NotHTML.__init__)__name__
__module____qualname__rK   __classcell__rB   rB   )rO   rC   rI   e   s   rI   c             C   s.   | j dd}| ds*t|| jjdS )zCheck the Content-Type header to ensure the response contains HTML.

    Raises `_NotHTML` if the content type is not text/html.
    zContent-Type z	text/htmlN)headersgetr=   r>   rI   r   method)responserL   rB   rB   rC   _ensure_html_headerm   s    rY   c               @   s   e Zd ZdS )_NotHTTPN)rP   rQ   rR   rB   rB   rB   rC   rZ   x   s   rZ   c             C   sD   t | \}}}}}|dkr"t |j| dd}|  t| dS )zSend a HEAD request to the URL, and ensure the response contains HTML.

    Raises `_NotHTTP` if the URL is not available for a HEAD request, or
    `_NotHTML` if the content type is not text/html.
    >   httpr5   T)allow_redirectsN)urllib_parseurlsplitrZ   headraise_for_statusrY   )r@   sessionrA   netlocpathqueryfragmentresprB   rB   rC   _ensure_html_response|   s    rg   c             C   sH   t | rt| |d td|  |j| dddd}|  t| |S )a  Access an HTML page with GET, and return the response.

    This consists of three parts:

    1. If the URL looks suspiciously like an archive, send a HEAD first to
       check the Content-Type is HTML, to avoid downloading a large file.
       Raise `_NotHTTP` if the content type cannot be determined, or
       `_NotHTML` if it is not HTML.
    2. Actually perform the request. Raise HTTP exceptions on network failures.
    3. Check the Content-Type header to make sure we got HTML, and raise
       `_NotHTML` otherwise.
    )ra   zGetting page %sz	text/htmlz	max-age=0)AcceptzCache-Control)rU   )rH   rg   loggerdebugrV   r`   rY   )r@   ra   rf   rB   rB   rC   _get_html_response   s    rk   c             C   s   |d krt j}|d| | d S )Nz%Could not fetch URL %s: %s - skipping)ri   rj   )linkreasonmethrB   rB   rC   _handle_get_page_fail   s    ro   c       
   
   C   s  |d krt d| jddd }t|}|r@td||  d S t|\}}}}}}|dkrtj	
t|r|ds|d7 }t|d}td	| yt||d
}W nV tk
r } ztd|  W d d }~X Y n: tk
r } ztd| |j|j W d d }~X Y n tjk
rB } zt| | W d d }~X Y n tk
rn } zt| | W d d }~X Y n tk
r } z$d}	|	t|7 }	t| |	tjd W d d }~X Y nf tjk
r } zt| d|  W d d }~X Y n4 tjk
r   t| d Y nX t|j|j|jS d S )Nz?_get_html_page() missing 1 required keyword argument: 'session'#   r   zCannot look at %s URL %sr8   /z
index.htmlz# file: URL is directory, getting %s)ra   zQSkipping page %s because it looks like an archive, and cannot be checked by HEAD.z<Skipping page %s because the %s request got Content-Type: %sz4There was a problem confirming the ssl certificate: )rn   zconnection error: %sz	timed out) 	TypeErrorr@   splitrD   ri   rj   r]   urlparseosrc   isdirurllib_requesturl2pathnamerF   urljoinrk   rZ   rI   rM   rL   r   	HTTPErrorro   r   r   strinfoConnectionErrorTimeoutHTMLPagecontentrU   )
rl   ra   r@   
vcs_schemerA   _rc   rf   excrm   rB   rB   rC   _get_html_page   sL    
  r   c            	   @   s   e Zd ZdZdddZdd Zed dd	Zd
d Zdd Z	dd Z
dd Zdd Zdd ZedZdd Zdd Zdd Zdd ZdS )!r4   zThis finds packages.

    This is meant to match easy_install's technique for looking for
    packages, by reading pages and looking for appropriate links.
    FNc             C   s   |dkrt dg | _x:|D ]2}|drBt|}tj|rB|}| j| qW || _t	 | _
|pptt	 t	 | _dd |r|ng D | _|| _|| _t|||	|
d| _|| _tsx8t| j| jD ]$}t|}|jdkrtd P qW dS )	a  Create a PackageFinder.

        :param format_control: A FormatControl object or None. Used to control
            the selection of source packages / binary packages when consulting
            the index and links.
        :param platform: A string or None. If None, searches for packages
            that are supported by the current system. Otherwise, will find
            packages that can be built on the platform passed in. These
            packages will only be downloaded for distribution: they will
            not be built locally.
        :param versions: A list of strings or None. This is passed directly
            to pep425tags.py in the get_supported() method.
        :param abi: A string or None. This is passed directly
            to pep425tags.py in the get_supported() method.
        :param implementation: A string or None. This is passed directly
            to pep425tags.py in the get_supported() method.
        Nz>PackageFinder() missing 1 required keyword argument: 'session'~c             S   s   g | ]}d |d fqS )r6   rB   ).0hostrB   rB   rC   
<listcomp>D  s   z*PackageFinder.__init__.<locals>.<listcomp>)versionsplatformabiimplr5   zipip is configured with locations that require TLS/SSL, however the ssl module in Python is not available.)rs   
find_linksr>   r    rv   rc   existsappend
index_urlssetlogged_linksr   format_controlsecure_originsallow_all_prereleasesra   r   
valid_tagsprefer_binaryr   	itertoolschainr]   ru   rA   ri   warning)rN   r   r   r   trusted_hostsra   r   r   r   r   implementationr   rl   new_linkparsedrB   rB   rC   rK     s<     	




zPackageFinder.__init__c             C   sb   g }| j r:| j tjgkr:|dddd | j D  | jrX|dd| j d|S )NzLooking in indexes: {}z, c             s   s   | ]}t |V  qd S )N)r!   )r   r@   rB   rB   rC   	<genexpr>l  s    z8PackageFinder.get_formatted_locations.<locals>.<genexpr>zLooking in links: {}
)r   r   
simple_urlr   formatjoinr   )rN   linesrB   rB   rC   get_formatted_locationsf  s    z%PackageFinder.get_formatted_locationsc                s  g  g  fdd}x| D ]}t j|}|d}|s>|r|rH|}nt|}t j|r|rt j|}xFt |D ]}|t j|| qxW q|r	| qt
d| qt j|r|| qt
d| qt|r	| qt
d| qW  fS )zt
        Sort locations into "files" (archives) and "urls", and return
        a pair of lists (files,urls)
        c                s8   t | }tj|ddd dkr*| n
 | d S )NF)strictr   z	text/html)r   	mimetypes
guess_typer   )rc   r@   )filesurlsrB   rC   	sort_path  s    z0PackageFinder._sort_locations.<locals>.sort_pathzfile:z)Path '{0}' is ignored: it is a directory.z:Url '%s' is ignored: it is neither a file nor a directory.zQUrl '%s' is ignored. It is either a non-existing path or lacks a specific scheme.)rv   rc   r   r>   r   rw   realpathlistdirr   r   ri   r   r   isfiler   )	locations
expand_dirr   r@   is_local_pathis_file_urlrc   itemrB   )r   r   rC   _sort_locationst  s>    




zPackageFinder._sort_locationsc       	      C   s   t | j}t }d}|jjrt|jj}|| jsBtd|j | j	rLd}|
| j }|jdk	rtd|j}| }t|d |d f}n| }||j||fS )a  
        Function used to generate link sort key for link tuples.
        The greater the return value, the more preferred it is.
        If not finding wheels, then sorted by version only.
        If finding wheels, then the sort order is by version, then:
          1. existing installs
          2. wheels ordered via Wheel.support_index_min(self.valid_tags)
          3. source archives
        If prefer_binary was set, then all wheels are sorted above sources.
        Note: it was considered to embed this logic into the Link
              comparison operators, but then different sdist links
              with the same version, would have to be considered equal
        r   zB%s is not a supported wheel for this platform. It can't be sorted.rq   Nz^(\d+)(.*)$)r?   r   tuplelocationis_wheelr$   rE   	supportedr   r   support_index_min	build_tagrematchgroupsintversion)	rN   	candidatesupport_numr   binary_preferencewheelprir   build_tag_groupsrB   rB   rC   _candidate_sort_key  s$    


z!PackageFinder._candidate_sort_keyc       	   	   C   sl  t t|}|j|j|jf}|d ddd }x t| j D ]}||d kr`|d dkr`q@yht	
t|d tjs|d d kr|d n|d d}t	t|d tjr|d n|d d}W nJ tk
r   |d r|d  |d  kr|d dkrw@Y nX ||kr q@|d |d krP|d dkrP|d d k	rPq@dS W |d	|j|j d
S )Nr   +rq   r6   utf8   TzThe repository located at %s is not a trusted or secure host and is being ignored. If this repository is available via HTTPS we recommend you use HTTPS instead, otherwise you may silence this warning and allow it anyway with '--trusted-host %s'.F)r]   ru   r|   rA   hostnameportrsplitSECURE_ORIGINSr   r   
ip_address
isinstancer   	text_typedecode
ip_network
ValueErrorr=   r   )	rN   ri   r   r   originprotocolsecure_originaddrnetworkrB   rB   rC   _validate_secure_origin  s>    

z%PackageFinder._validate_secure_originc                s    fdd  fdd| j D S )zReturns the locations found via self.index_urls

        Checks the url_name on the main (first in the list) index and
        use this url_name to produce all locations
        c                s,   t | tt }|ds(|d }|S )Nrr   )	posixpathr   r]   quoter	   rF   )r@   loc)project_namerB   rC   mkurl_pypi_url%  s    
z?PackageFinder._get_index_urls_locations.<locals>.mkurl_pypi_urlc                s   g | ]} |qS rB   rB   )r   r@   )r   rB   rC   r   2  s    z;PackageFinder._get_index_urls_locations.<locals>.<listcomp>)r   )rN   r   rB   )r   r   rC   _get_index_urls_locations  s    z'PackageFinder._get_index_urls_locationsc          
      sj    |} |\}} j jdd\}}dd t||D } fddtdd |D dd |D D }td	t|| x|D ]}	td
|	 qW t|}
 j	
|
}t||
|} dd  jD |}g }xJ ||D ]:}td|j t  | | | W dQ R X qW  ||}|r^|jdd tdddd |D  || | S )a  Find all available InstallationCandidate for project_name

        This checks index_urls and find_links.
        All versions found are returned as an InstallationCandidate list.

        See _link_package_versions for details on which files are accepted
        T)r   c             s   s   | ]}t |V  qd S )N)r   )r   r@   rB   rB   rC   r   C  s    z4PackageFinder.find_all_candidates.<locals>.<genexpr>c                s   g | ]}  t|r|qS rB   )r   ri   )r   rl   )rN   rB   rC   r   K  s    z5PackageFinder.find_all_candidates.<locals>.<listcomp>c             s   s   | ]}t |V  qd S )N)r   )r   r@   rB   rB   rC   r   L  s    c             s   s   | ]}t |V  qd S )N)r   )r   r@   rB   rB   rC   r   M  s    z,%d location(s) to search for versions of %s:z* %sc             s   s   | ]}t |d V  qdS )z-fN)r   )r   r@   rB   rB   rC   r   ]  s    zAnalyzing links from page %sN)reversezLocal files found: %sz, c             S   s   g | ]}t |jjqS rB   )r   r   r@   )r   r   rB   rB   rC   r   o  s   )r   r   r   r   r   ri   rj   r?   r	   r   get_allowed_formatsSearch_package_versions
_get_pagesr@   r   extend
iter_linkssortr   )rN   r   index_locationsindex_file_locindex_url_locfl_file_loc
fl_url_locfile_locationsurl_locationsr   canonical_nameformatssearchfind_links_versionspage_versionspagefile_versionsrB   )rN   rC   find_all_candidates4  sD    	


 z!PackageFinder.find_all_candidatesc          
      sb  |  |j}t|jjdd |D | jr,| jndd  fdd|D }|r\t|| jd}nd}|jdk	rxt	|jj
}nd}|dkr|dkrtd|dtd	d
 |D t	d td| d}|r|dks|j
|krd}|s|dk	r|rtd| ntd||j
 dS |r>td|dt t	dp6d ttd|j
dt t	d |jS )zTry to find a Link matching req

        Expects req, an InstallRequirement and upgrade, a boolean
        Returns a Link if found,
        Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise
        c             S   s   g | ]}t |jqS rB   )r|   r   )r   crB   rB   rC   r     s    z2PackageFinder.find_requirement.<locals>.<listcomp>N)prereleasesc                s   g | ]}t |j kr|qS rB   )r|   r   )r   r   )compatible_versionsrB   rC   r     s    )keyzNCould not find a version that satisfies the requirement %s (from versions: %s)z, c             S   s   h | ]}t |jqS rB   )r|   r   )r   r   rB   rB   rC   	<setcomp>  s    z1PackageFinder.find_requirement.<locals>.<setcomp>z%No matching distribution found for %sFTzLExisting installed version (%s) is most up-to-date and satisfies requirementzUExisting installed version (%s) satisfies requirement (most up-to-date version is %s)z=Installed version (%s) is most up-to-date (past versions: %s)nonez)Using version %s (newest of versions: %s))r   namer   	specifierfilterr   maxr   satisfied_byparse_versionr   ri   criticalr   sortedr   rj   r   r   )rN   requpgradeall_candidatesapplicable_candidatesbest_candidateinstalled_versionbest_installedrB   )r   rC   find_requirementw  sd    


zPackageFinder.find_requirementc             c   sJ   t  }x>|D ]6}||krq|| t|| jd}|dkr<q|V  qW dS )zp
        Yields (page, page_url) from the given locations, skipping
        locations that have errors.
        )ra   N)r   addr   ra   )rN   r   r   seenr   r   rB   rB   rC   r     s    

zPackageFinder._get_pagesz-py([123]\.?[0-9]?)$c             C   sT   g g  }}t  }x:|D ]2}||kr|| |jr>|| q|| qW || S )z
        Returns elements of links in order, non-egg links first, egg links
        second, while eliminating duplicates
        )r   r  egg_fragmentr   )rN   linkseggsno_eggsr  rl   rB   rB   rC   _sort_links  s    


zPackageFinder._sort_linksc             C   s:   g }x0|  |D ]"}| ||}|d k	r|| qW |S )N)r  _link_package_versionsr   )rN   r  r   resultrl   vrB   rB   rC   r      s    zPackageFinder._package_versionsc             C   s(   || j kr$td|| | j | d S )NzSkipping link %s; %s)r   ri   rj   r  )rN   rl   rm   rB   rB   rC   _log_skipped_link  s    
zPackageFinder._log_skipped_linkc       
      C   sD  d}|j r|j }|j}n| \}}|s:| |d dS |tkrV| |d|  dS d|jkr~|tkr~| |d|j  dS d|jkr|dkr| |d dS |tkr&yt	|j
}W n  tk
r   | |d	 dS X t|j|jkr| |d
|j  dS || js | |d dS |j}d|jkrR|tkrR| |d|j  dS |sdt||j}|s| |d|j  dS | j|}|r|d|  }|d}|tjdd kr| |d dS yt|j}	W n. tjk
r   td|j
|j d}	Y nX |	s(td||j dS td|| t|j||S )z'Return an InstallationCandidate or NoneNz
not a filezunsupported archive format: %sbinaryzNo binaries permitted for %smacosx10z.zipzmacosx10 onezinvalid wheel filenamezwrong project name (not %s)z%it is not compatible with this PythonsourcezNo sources permitted for %szMissing project version for %srq      zPython version is incorrectz3Package %s has an invalid Requires-Python entry: %sTzaThe package %s is incompatible with the python version in use. Acceptable python versions are: %szFound link %s, version: %s)r  extsplitextr  r   r   r   suppliedrc   r$   rE   r   r	   r   	canonicalr   r   r   _egg_info_matches_py_version_rer   startgroupsysr"   requires_pythonr   InvalidSpecifierri   rj   r   )
rN   rl   r   r   egg_infor  r   r   
py_versionsupport_this_pythonrB   rB   rC   r    s    




z$PackageFinder._link_package_versions)	FNNNNNNNF)F)rP   rQ   rR   __doc__rK   r   staticmethodr   r   r   r   r   r  r   r   compiler#  r  r   r  r  rB   rB   rB   rC   r4     s0           
R9%JCd
c             C   sL   x6t | D ]*\}}|dkrq
t| d| |kr
|S q
W td| |dS )a  Find the separator's index based on the package's canonical name.

    `egg_info` must be an egg info string for the given package, and
    `canonical_name` must be the package's canonical name.

    This function is needed since the canonicalized name does not necessarily
    have the same length as the egg info's name part. An example::

    >>> egg_info = 'foo__bar-1.0'
    >>> canonical_name = 'foo-bar'
    >>> _find_name_version_sep(egg_info, canonical_name)
    8
    -Nz{} does not match {})	enumerater	   r   r   )r)  r   ir   rB   rB   rC   _find_name_version_sepe  s    r2  c             C   s@   yt | |d }W n tk
r&   dS X | |d }|s<dS |S )zPull the version part out of a string.

    :param egg_info: The string to parse. E.g. foo-2.1
    :param canonical_name: The canonicalized name of the package this
        belongs to.
    rq   N)r2  r   )r)  r   version_startr   rB   rB   rC   r"    s    r"  c             C   s.   x(|  dD ]}|d}|dk	r|S qW |S )a  Determine the HTML document's base URL.

    This looks for a ``<base>`` tag in the HTML document. If present, its href
    attribute denotes the base URL of anchor tags in the document. If there is
    no such tag (or if it does not have a valid href attribute), the HTML
    file's URL is used as the base URL.

    :param document: An HTML document representation. The current
        implementation expects the result of ``html5lib.parse()``.
    :param page_url: The URL of the HTML document.
    z.//basehrefN)findallrV   )documentpage_urlbaser4  rB   rB   rC   _determine_base_url  s
    
r9  c             C   s2   | r.d| kr.t | d \}}d|kr.|d S dS )zBDetermine if we have any encoding information in our headers.
    zContent-TypecharsetN)cgiparse_header)rU   rL   paramsrB   rB   rC   _get_encoding_from_headers  s
    r>  z[^a-z0-9$&+,/:;=?@.#%_\\|-]c             C   s   t dd | S )zMakes sure a link is fully encoded.  That is, if a ' ' shows up in
    the link, it will be rewritten to %20 (while not over-quoting
    % or other characters).c             S   s   dt | d S )Nz%%%2xr   )ordr%  )r   rB   rB   rC   <lambda>      z_clean_link.<locals>.<lambda>)_CLEAN_LINK_REsub)r@   rB   rB   rC   _clean_link  s    rD  c               @   s*   e Zd ZdZd	ddZdd Zdd ZdS )
r   z'Represents one page, along with its URLNc             C   s   || _ || _|| _d S )N)r   r@   rU   )rN   r   r@   rU   rB   rB   rC   rK     s    zHTMLPage.__init__c             C   s
   t | jS )N)r!   r@   )rN   rB   rB   rC   __str__  s    zHTMLPage.__str__c             c   s   t j| jt| jdd}t|| j}xb|dD ]T}|dr0|d}t	t
||}|d}|rnt|nd}t|| j|dV  q0W dS )zYields all links in the pageF)transport_encodingnamespaceHTMLElementsz.//ar4  zdata-requires-pythonN)r'  )r   r
   r   r>  rU   r9  r@   r5  rV   rD  r]   rz   r   r   )rN   r6  base_urlanchorr4  r@   	pyrequirerB   rB   rC   r     s    


zHTMLPage.iter_links)N)rP   rQ   rR   r,  rK   rE  r   rB   rB   rB   rC   r     s   
r   r   zsupplied canonical formats)N)N)wr,  
__future__r   r;  r   loggingr   rv   r   r   r&  collectionsr   pip._vendorr   r   r   pip._vendor.distlib.compatr   Zpip._vendor.packagingr   pip._vendor.packaging.utilsr	   Zpip._vendor.packaging.versionr
   r  Zpip._vendor.requests.exceptionsr   r   pip._vendor.six.moves.urllibr]   r   rx   pip._internal.downloadr   r   r   r   pip._internal.exceptionsr   r   r   r   pip._internal.models.candidater   #pip._internal.models.format_controlr   pip._internal.models.indexr   pip._internal.models.linkr   pip._internal.pep425tagsr   pip._internal.utils.compatr   pip._internal.utils.loggingr   pip._internal.utils.miscr   r   r   r    r!   pip._internal.utils.packagingr"   pip._internal.utils.typingr#   pip._internal.wheelr$   r%   typingr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   Zpip._vendor.requestsr1   Zpip._internal.reqr2   r3   r|   ZSecureOriginZBuildTagr   ZCandidateSortingKey__all__r   	getLoggerrP   ri   rD   rH   	ExceptionrI   rY   rZ   rg   rk   ro   r   objectr4   r2  r"  r9  r>  r.  IrB  rD  r   r   rB   rB   rB   rC   <module>   s   0
6

6    h

