Add loan-advisor-agent
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__init__.py +211 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_base_connection.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_collections.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_request_methods.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_version.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/connection.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/connectionpool.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/exceptions.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/fields.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/filepost.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/poolmanager.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/response.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/_base_connection.py +165 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/_collections.py +487 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/_request_methods.py +278 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/_version.py +34 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/connection.py +1099 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/connectionpool.py +1178 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__pycache__/socks.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/__init__.py +17 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/connection.py +260 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/emscripten_fetch_worker.js +110 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/fetch.py +726 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/request.py +22 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/response.py +277 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/pyopenssl.py +564 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/socks.py +228 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/exceptions.py +335 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/fields.py +341 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/filepost.py +89 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__init__.py +53 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__pycache__/probe.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/connection.py +356 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/probe.py +87 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/poolmanager.py +651 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/py.typed +2 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/response.py +1480 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__init__.py +42 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/__init__.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/connection.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/proxy.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/request.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/response.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/retry.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/ssl_.cpython-312.pyc +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/ssl_match_hostname.cpython-312.pyc +0 -0
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__init__.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
# Set default logging handler to avoid "No handler found" warnings.
|
| 8 |
+
import logging
|
| 9 |
+
import sys
|
| 10 |
+
import typing
|
| 11 |
+
import warnings
|
| 12 |
+
from logging import NullHandler
|
| 13 |
+
|
| 14 |
+
from . import exceptions
|
| 15 |
+
from ._base_connection import _TYPE_BODY
|
| 16 |
+
from ._collections import HTTPHeaderDict
|
| 17 |
+
from ._version import __version__
|
| 18 |
+
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
|
| 19 |
+
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
| 20 |
+
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
|
| 21 |
+
from .response import BaseHTTPResponse, HTTPResponse
|
| 22 |
+
from .util.request import make_headers
|
| 23 |
+
from .util.retry import Retry
|
| 24 |
+
from .util.timeout import Timeout
|
| 25 |
+
|
| 26 |
+
# Ensure that Python is compiled with OpenSSL 1.1.1+
|
| 27 |
+
# If the 'ssl' module isn't available at all that's
|
| 28 |
+
# fine, we only care if the module is available.
|
| 29 |
+
try:
|
| 30 |
+
import ssl
|
| 31 |
+
except ImportError:
|
| 32 |
+
pass
|
| 33 |
+
else:
|
| 34 |
+
if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
|
| 35 |
+
warnings.warn(
|
| 36 |
+
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
| 37 |
+
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
| 38 |
+
"See: https://github.com/urllib3/urllib3/issues/3020",
|
| 39 |
+
exceptions.NotOpenSSLWarning,
|
| 40 |
+
)
|
| 41 |
+
elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
|
| 42 |
+
raise ImportError(
|
| 43 |
+
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
| 44 |
+
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
| 45 |
+
"See: https://github.com/urllib3/urllib3/issues/2168"
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
|
| 49 |
+
__license__ = "MIT"
|
| 50 |
+
__version__ = __version__
|
| 51 |
+
|
| 52 |
+
__all__ = (
|
| 53 |
+
"HTTPConnectionPool",
|
| 54 |
+
"HTTPHeaderDict",
|
| 55 |
+
"HTTPSConnectionPool",
|
| 56 |
+
"PoolManager",
|
| 57 |
+
"ProxyManager",
|
| 58 |
+
"HTTPResponse",
|
| 59 |
+
"Retry",
|
| 60 |
+
"Timeout",
|
| 61 |
+
"add_stderr_logger",
|
| 62 |
+
"connection_from_url",
|
| 63 |
+
"disable_warnings",
|
| 64 |
+
"encode_multipart_formdata",
|
| 65 |
+
"make_headers",
|
| 66 |
+
"proxy_from_url",
|
| 67 |
+
"request",
|
| 68 |
+
"BaseHTTPResponse",
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
logging.getLogger(__name__).addHandler(NullHandler())
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def add_stderr_logger(
|
| 75 |
+
level: int = logging.DEBUG,
|
| 76 |
+
) -> logging.StreamHandler[typing.TextIO]:
|
| 77 |
+
"""
|
| 78 |
+
Helper for quickly adding a StreamHandler to the logger. Useful for
|
| 79 |
+
debugging.
|
| 80 |
+
|
| 81 |
+
Returns the handler after adding it.
|
| 82 |
+
"""
|
| 83 |
+
# This method needs to be in this __init__.py to get the __name__ correct
|
| 84 |
+
# even if urllib3 is vendored within another package.
|
| 85 |
+
logger = logging.getLogger(__name__)
|
| 86 |
+
handler = logging.StreamHandler()
|
| 87 |
+
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
| 88 |
+
logger.addHandler(handler)
|
| 89 |
+
logger.setLevel(level)
|
| 90 |
+
logger.debug("Added a stderr logging handler to logger: %s", __name__)
|
| 91 |
+
return handler
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# ... Clean up.
|
| 95 |
+
del NullHandler
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
# All warning filters *must* be appended unless you're really certain that they
|
| 99 |
+
# shouldn't be: otherwise, it's very hard for users to use most Python
|
| 100 |
+
# mechanisms to silence them.
|
| 101 |
+
# SecurityWarning's always go off by default.
|
| 102 |
+
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
|
| 103 |
+
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
|
| 104 |
+
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
|
| 108 |
+
"""
|
| 109 |
+
Helper for quickly disabling all urllib3 warnings.
|
| 110 |
+
"""
|
| 111 |
+
warnings.simplefilter("ignore", category)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
_DEFAULT_POOL = PoolManager()
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def request(
|
| 118 |
+
method: str,
|
| 119 |
+
url: str,
|
| 120 |
+
*,
|
| 121 |
+
body: _TYPE_BODY | None = None,
|
| 122 |
+
fields: _TYPE_FIELDS | None = None,
|
| 123 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 124 |
+
preload_content: bool | None = True,
|
| 125 |
+
decode_content: bool | None = True,
|
| 126 |
+
redirect: bool | None = True,
|
| 127 |
+
retries: Retry | bool | int | None = None,
|
| 128 |
+
timeout: Timeout | float | int | None = 3,
|
| 129 |
+
json: typing.Any | None = None,
|
| 130 |
+
) -> BaseHTTPResponse:
|
| 131 |
+
"""
|
| 132 |
+
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
|
| 133 |
+
Therefore, its side effects could be shared across dependencies relying on it.
|
| 134 |
+
To avoid side effects create a new ``PoolManager`` instance and use it instead.
|
| 135 |
+
The method does not accept low-level ``**urlopen_kw`` keyword arguments.
|
| 136 |
+
|
| 137 |
+
:param method:
|
| 138 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 139 |
+
|
| 140 |
+
:param url:
|
| 141 |
+
The URL to perform the request on.
|
| 142 |
+
|
| 143 |
+
:param body:
|
| 144 |
+
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
| 145 |
+
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
| 146 |
+
|
| 147 |
+
:param fields:
|
| 148 |
+
Data to encode and send in the request body.
|
| 149 |
+
|
| 150 |
+
:param headers:
|
| 151 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 152 |
+
If-None-Match, etc.
|
| 153 |
+
|
| 154 |
+
:param bool preload_content:
|
| 155 |
+
If True, the response's body will be preloaded into memory.
|
| 156 |
+
|
| 157 |
+
:param bool decode_content:
|
| 158 |
+
If True, will attempt to decode the body based on the
|
| 159 |
+
'content-encoding' header.
|
| 160 |
+
|
| 161 |
+
:param redirect:
|
| 162 |
+
If True, automatically handle redirects (status codes 301, 302,
|
| 163 |
+
303, 307, 308). Each redirect counts as a retry. Disabling retries
|
| 164 |
+
will disable redirect, too.
|
| 165 |
+
|
| 166 |
+
:param retries:
|
| 167 |
+
Configure the number of retries to allow before raising a
|
| 168 |
+
:class:`~urllib3.exceptions.MaxRetryError` exception.
|
| 169 |
+
|
| 170 |
+
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
|
| 171 |
+
:class:`~urllib3.util.retry.Retry` object for fine-grained control
|
| 172 |
+
over different types of retries.
|
| 173 |
+
Pass an integer number to retry connection errors that many times,
|
| 174 |
+
but no other types of errors. Pass zero to never retry.
|
| 175 |
+
|
| 176 |
+
If ``False``, then retries are disabled and any exception is raised
|
| 177 |
+
immediately. Also, instead of raising a MaxRetryError on redirects,
|
| 178 |
+
the redirect response will be returned.
|
| 179 |
+
|
| 180 |
+
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
|
| 181 |
+
|
| 182 |
+
:param timeout:
|
| 183 |
+
If specified, overrides the default timeout for this one
|
| 184 |
+
request. It may be a float (in seconds) or an instance of
|
| 185 |
+
:class:`urllib3.util.Timeout`.
|
| 186 |
+
|
| 187 |
+
:param json:
|
| 188 |
+
Data to encode and send as JSON with UTF-encoded in the request body.
|
| 189 |
+
The ``"Content-Type"`` header will be set to ``"application/json"``
|
| 190 |
+
unless specified otherwise.
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
return _DEFAULT_POOL.request(
|
| 194 |
+
method,
|
| 195 |
+
url,
|
| 196 |
+
body=body,
|
| 197 |
+
fields=fields,
|
| 198 |
+
headers=headers,
|
| 199 |
+
preload_content=preload_content,
|
| 200 |
+
decode_content=decode_content,
|
| 201 |
+
redirect=redirect,
|
| 202 |
+
retries=retries,
|
| 203 |
+
timeout=timeout,
|
| 204 |
+
json=json,
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
if sys.platform == "emscripten":
|
| 209 |
+
from .contrib.emscripten import inject_into_urllib3 # noqa: 401
|
| 210 |
+
|
| 211 |
+
inject_into_urllib3()
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (7.31 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_base_connection.cpython-312.pyc
ADDED
|
Binary file (6.86 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_collections.cpython-312.pyc
ADDED
|
Binary file (22.9 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_request_methods.cpython-312.pyc
ADDED
|
Binary file (10.6 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/_version.cpython-312.pyc
ADDED
|
Binary file (792 Bytes). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/connection.cpython-312.pyc
ADDED
|
Binary file (38.9 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/connectionpool.cpython-312.pyc
ADDED
|
Binary file (39.7 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/exceptions.cpython-312.pyc
ADDED
|
Binary file (16.6 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/fields.cpython-312.pyc
ADDED
|
Binary file (12 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/filepost.cpython-312.pyc
ADDED
|
Binary file (3.48 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/poolmanager.cpython-312.pyc
ADDED
|
Binary file (24.3 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/__pycache__/response.cpython-312.pyc
ADDED
|
Binary file (57.5 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/_base_connection.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import typing
|
| 4 |
+
|
| 5 |
+
from .util.connection import _TYPE_SOCKET_OPTIONS
|
| 6 |
+
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
| 7 |
+
from .util.url import Url
|
| 8 |
+
|
| 9 |
+
_TYPE_BODY = typing.Union[bytes, typing.IO[typing.Any], typing.Iterable[bytes], str]
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ProxyConfig(typing.NamedTuple):
|
| 13 |
+
ssl_context: ssl.SSLContext | None
|
| 14 |
+
use_forwarding_for_https: bool
|
| 15 |
+
assert_hostname: None | str | typing.Literal[False]
|
| 16 |
+
assert_fingerprint: str | None
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class _ResponseOptions(typing.NamedTuple):
|
| 20 |
+
# TODO: Remove this in favor of a better
|
| 21 |
+
# HTTP request/response lifecycle tracking.
|
| 22 |
+
request_method: str
|
| 23 |
+
request_url: str
|
| 24 |
+
preload_content: bool
|
| 25 |
+
decode_content: bool
|
| 26 |
+
enforce_content_length: bool
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
if typing.TYPE_CHECKING:
|
| 30 |
+
import ssl
|
| 31 |
+
from typing import Protocol
|
| 32 |
+
|
| 33 |
+
from .response import BaseHTTPResponse
|
| 34 |
+
|
| 35 |
+
class BaseHTTPConnection(Protocol):
|
| 36 |
+
default_port: typing.ClassVar[int]
|
| 37 |
+
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
| 38 |
+
|
| 39 |
+
host: str
|
| 40 |
+
port: int
|
| 41 |
+
timeout: None | (
|
| 42 |
+
float
|
| 43 |
+
) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
|
| 44 |
+
blocksize: int
|
| 45 |
+
source_address: tuple[str, int] | None
|
| 46 |
+
socket_options: _TYPE_SOCKET_OPTIONS | None
|
| 47 |
+
|
| 48 |
+
proxy: Url | None
|
| 49 |
+
proxy_config: ProxyConfig | None
|
| 50 |
+
|
| 51 |
+
is_verified: bool
|
| 52 |
+
proxy_is_verified: bool | None
|
| 53 |
+
|
| 54 |
+
def __init__(
|
| 55 |
+
self,
|
| 56 |
+
host: str,
|
| 57 |
+
port: int | None = None,
|
| 58 |
+
*,
|
| 59 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 60 |
+
source_address: tuple[str, int] | None = None,
|
| 61 |
+
blocksize: int = 8192,
|
| 62 |
+
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
| 63 |
+
proxy: Url | None = None,
|
| 64 |
+
proxy_config: ProxyConfig | None = None,
|
| 65 |
+
) -> None: ...
|
| 66 |
+
|
| 67 |
+
def set_tunnel(
|
| 68 |
+
self,
|
| 69 |
+
host: str,
|
| 70 |
+
port: int | None = None,
|
| 71 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 72 |
+
scheme: str = "http",
|
| 73 |
+
) -> None: ...
|
| 74 |
+
|
| 75 |
+
def connect(self) -> None: ...
|
| 76 |
+
|
| 77 |
+
def request(
|
| 78 |
+
self,
|
| 79 |
+
method: str,
|
| 80 |
+
url: str,
|
| 81 |
+
body: _TYPE_BODY | None = None,
|
| 82 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 83 |
+
# We know *at least* botocore is depending on the order of the
|
| 84 |
+
# first 3 parameters so to be safe we only mark the later ones
|
| 85 |
+
# as keyword-only to ensure we have space to extend.
|
| 86 |
+
*,
|
| 87 |
+
chunked: bool = False,
|
| 88 |
+
preload_content: bool = True,
|
| 89 |
+
decode_content: bool = True,
|
| 90 |
+
enforce_content_length: bool = True,
|
| 91 |
+
) -> None: ...
|
| 92 |
+
|
| 93 |
+
def getresponse(self) -> BaseHTTPResponse: ...
|
| 94 |
+
|
| 95 |
+
def close(self) -> None: ...
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def is_closed(self) -> bool:
|
| 99 |
+
"""Whether the connection either is brand new or has been previously closed.
|
| 100 |
+
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
|
| 101 |
+
properties must be False.
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
@property
|
| 105 |
+
def is_connected(self) -> bool:
|
| 106 |
+
"""Whether the connection is actively connected to any origin (proxy or target)"""
|
| 107 |
+
|
| 108 |
+
@property
|
| 109 |
+
def has_connected_to_proxy(self) -> bool:
|
| 110 |
+
"""Whether the connection has successfully connected to its proxy.
|
| 111 |
+
This returns False if no proxy is in use. Used to determine whether
|
| 112 |
+
errors are coming from the proxy layer or from tunnelling to the target origin.
|
| 113 |
+
"""
|
| 114 |
+
|
| 115 |
+
class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
|
| 116 |
+
default_port: typing.ClassVar[int]
|
| 117 |
+
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
| 118 |
+
|
| 119 |
+
# Certificate verification methods
|
| 120 |
+
cert_reqs: int | str | None
|
| 121 |
+
assert_hostname: None | str | typing.Literal[False]
|
| 122 |
+
assert_fingerprint: str | None
|
| 123 |
+
ssl_context: ssl.SSLContext | None
|
| 124 |
+
|
| 125 |
+
# Trusted CAs
|
| 126 |
+
ca_certs: str | None
|
| 127 |
+
ca_cert_dir: str | None
|
| 128 |
+
ca_cert_data: None | str | bytes
|
| 129 |
+
|
| 130 |
+
# TLS version
|
| 131 |
+
ssl_minimum_version: int | None
|
| 132 |
+
ssl_maximum_version: int | None
|
| 133 |
+
ssl_version: int | str | None # Deprecated
|
| 134 |
+
|
| 135 |
+
# Client certificates
|
| 136 |
+
cert_file: str | None
|
| 137 |
+
key_file: str | None
|
| 138 |
+
key_password: str | None
|
| 139 |
+
|
| 140 |
+
def __init__(
|
| 141 |
+
self,
|
| 142 |
+
host: str,
|
| 143 |
+
port: int | None = None,
|
| 144 |
+
*,
|
| 145 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 146 |
+
source_address: tuple[str, int] | None = None,
|
| 147 |
+
blocksize: int = 16384,
|
| 148 |
+
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
| 149 |
+
proxy: Url | None = None,
|
| 150 |
+
proxy_config: ProxyConfig | None = None,
|
| 151 |
+
cert_reqs: int | str | None = None,
|
| 152 |
+
assert_hostname: None | str | typing.Literal[False] = None,
|
| 153 |
+
assert_fingerprint: str | None = None,
|
| 154 |
+
server_hostname: str | None = None,
|
| 155 |
+
ssl_context: ssl.SSLContext | None = None,
|
| 156 |
+
ca_certs: str | None = None,
|
| 157 |
+
ca_cert_dir: str | None = None,
|
| 158 |
+
ca_cert_data: None | str | bytes = None,
|
| 159 |
+
ssl_minimum_version: int | None = None,
|
| 160 |
+
ssl_maximum_version: int | None = None,
|
| 161 |
+
ssl_version: int | str | None = None, # Deprecated
|
| 162 |
+
cert_file: str | None = None,
|
| 163 |
+
key_file: str | None = None,
|
| 164 |
+
key_password: str | None = None,
|
| 165 |
+
) -> None: ...
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/_collections.py
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import typing
|
| 4 |
+
from collections import OrderedDict
|
| 5 |
+
from enum import Enum, auto
|
| 6 |
+
from threading import RLock
|
| 7 |
+
|
| 8 |
+
if typing.TYPE_CHECKING:
|
| 9 |
+
# We can only import Protocol if TYPE_CHECKING because it's a development
|
| 10 |
+
# dependency, and is not available at runtime.
|
| 11 |
+
from typing import Protocol
|
| 12 |
+
|
| 13 |
+
from typing_extensions import Self
|
| 14 |
+
|
| 15 |
+
class HasGettableStringKeys(Protocol):
|
| 16 |
+
def keys(self) -> typing.Iterator[str]: ...
|
| 17 |
+
|
| 18 |
+
def __getitem__(self, key: str) -> str: ...
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
# Key type
|
| 25 |
+
_KT = typing.TypeVar("_KT")
|
| 26 |
+
# Value type
|
| 27 |
+
_VT = typing.TypeVar("_VT")
|
| 28 |
+
# Default type
|
| 29 |
+
_DT = typing.TypeVar("_DT")
|
| 30 |
+
|
| 31 |
+
ValidHTTPHeaderSource = typing.Union[
|
| 32 |
+
"HTTPHeaderDict",
|
| 33 |
+
typing.Mapping[str, str],
|
| 34 |
+
typing.Iterable[tuple[str, str]],
|
| 35 |
+
"HasGettableStringKeys",
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class _Sentinel(Enum):
|
| 40 |
+
not_passed = auto()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def ensure_can_construct_http_header_dict(
|
| 44 |
+
potential: object,
|
| 45 |
+
) -> ValidHTTPHeaderSource | None:
|
| 46 |
+
if isinstance(potential, HTTPHeaderDict):
|
| 47 |
+
return potential
|
| 48 |
+
elif isinstance(potential, typing.Mapping):
|
| 49 |
+
# Full runtime checking of the contents of a Mapping is expensive, so for the
|
| 50 |
+
# purposes of typechecking, we assume that any Mapping is the right shape.
|
| 51 |
+
return typing.cast(typing.Mapping[str, str], potential)
|
| 52 |
+
elif isinstance(potential, typing.Iterable):
|
| 53 |
+
# Similarly to Mapping, full runtime checking of the contents of an Iterable is
|
| 54 |
+
# expensive, so for the purposes of typechecking, we assume that any Iterable
|
| 55 |
+
# is the right shape.
|
| 56 |
+
return typing.cast(typing.Iterable[tuple[str, str]], potential)
|
| 57 |
+
elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
|
| 58 |
+
return typing.cast("HasGettableStringKeys", potential)
|
| 59 |
+
else:
|
| 60 |
+
return None
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
|
| 64 |
+
"""
|
| 65 |
+
Provides a thread-safe dict-like container which maintains up to
|
| 66 |
+
``maxsize`` keys while throwing away the least-recently-used keys beyond
|
| 67 |
+
``maxsize``.
|
| 68 |
+
|
| 69 |
+
:param maxsize:
|
| 70 |
+
Maximum number of recent elements to retain.
|
| 71 |
+
|
| 72 |
+
:param dispose_func:
|
| 73 |
+
Every time an item is evicted from the container,
|
| 74 |
+
``dispose_func(value)`` is called. Callback which will get called
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
_container: typing.OrderedDict[_KT, _VT]
|
| 78 |
+
_maxsize: int
|
| 79 |
+
dispose_func: typing.Callable[[_VT], None] | None
|
| 80 |
+
lock: RLock
|
| 81 |
+
|
| 82 |
+
def __init__(
|
| 83 |
+
self,
|
| 84 |
+
maxsize: int = 10,
|
| 85 |
+
dispose_func: typing.Callable[[_VT], None] | None = None,
|
| 86 |
+
) -> None:
|
| 87 |
+
super().__init__()
|
| 88 |
+
self._maxsize = maxsize
|
| 89 |
+
self.dispose_func = dispose_func
|
| 90 |
+
self._container = OrderedDict()
|
| 91 |
+
self.lock = RLock()
|
| 92 |
+
|
| 93 |
+
def __getitem__(self, key: _KT) -> _VT:
|
| 94 |
+
# Re-insert the item, moving it to the end of the eviction line.
|
| 95 |
+
with self.lock:
|
| 96 |
+
item = self._container.pop(key)
|
| 97 |
+
self._container[key] = item
|
| 98 |
+
return item
|
| 99 |
+
|
| 100 |
+
def __setitem__(self, key: _KT, value: _VT) -> None:
|
| 101 |
+
evicted_item = None
|
| 102 |
+
with self.lock:
|
| 103 |
+
# Possibly evict the existing value of 'key'
|
| 104 |
+
try:
|
| 105 |
+
# If the key exists, we'll overwrite it, which won't change the
|
| 106 |
+
# size of the pool. Because accessing a key should move it to
|
| 107 |
+
# the end of the eviction line, we pop it out first.
|
| 108 |
+
evicted_item = key, self._container.pop(key)
|
| 109 |
+
self._container[key] = value
|
| 110 |
+
except KeyError:
|
| 111 |
+
# When the key does not exist, we insert the value first so that
|
| 112 |
+
# evicting works in all cases, including when self._maxsize is 0
|
| 113 |
+
self._container[key] = value
|
| 114 |
+
if len(self._container) > self._maxsize:
|
| 115 |
+
# If we didn't evict an existing value, and we've hit our maximum
|
| 116 |
+
# size, then we have to evict the least recently used item from
|
| 117 |
+
# the beginning of the container.
|
| 118 |
+
evicted_item = self._container.popitem(last=False)
|
| 119 |
+
|
| 120 |
+
# After releasing the lock on the pool, dispose of any evicted value.
|
| 121 |
+
if evicted_item is not None and self.dispose_func:
|
| 122 |
+
_, evicted_value = evicted_item
|
| 123 |
+
self.dispose_func(evicted_value)
|
| 124 |
+
|
| 125 |
+
def __delitem__(self, key: _KT) -> None:
|
| 126 |
+
with self.lock:
|
| 127 |
+
value = self._container.pop(key)
|
| 128 |
+
|
| 129 |
+
if self.dispose_func:
|
| 130 |
+
self.dispose_func(value)
|
| 131 |
+
|
| 132 |
+
def __len__(self) -> int:
|
| 133 |
+
with self.lock:
|
| 134 |
+
return len(self._container)
|
| 135 |
+
|
| 136 |
+
def __iter__(self) -> typing.NoReturn:
|
| 137 |
+
raise NotImplementedError(
|
| 138 |
+
"Iteration over this class is unlikely to be threadsafe."
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
def clear(self) -> None:
|
| 142 |
+
with self.lock:
|
| 143 |
+
# Copy pointers to all values, then wipe the mapping
|
| 144 |
+
values = list(self._container.values())
|
| 145 |
+
self._container.clear()
|
| 146 |
+
|
| 147 |
+
if self.dispose_func:
|
| 148 |
+
for value in values:
|
| 149 |
+
self.dispose_func(value)
|
| 150 |
+
|
| 151 |
+
def keys(self) -> set[_KT]: # type: ignore[override]
|
| 152 |
+
with self.lock:
|
| 153 |
+
return set(self._container.keys())
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
class HTTPHeaderDictItemView(set[tuple[str, str]]):
|
| 157 |
+
"""
|
| 158 |
+
HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
|
| 159 |
+
address.
|
| 160 |
+
|
| 161 |
+
If we directly try to get an item with a particular name, we will get a string
|
| 162 |
+
back that is the concatenated version of all the values:
|
| 163 |
+
|
| 164 |
+
>>> d['X-Header-Name']
|
| 165 |
+
'Value1, Value2, Value3'
|
| 166 |
+
|
| 167 |
+
However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
|
| 168 |
+
these values based on whether combine=True was called when building up the dictionary
|
| 169 |
+
|
| 170 |
+
>>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
|
| 171 |
+
>>> d.add("A", "2", combine=True)
|
| 172 |
+
>>> d.add("B", "bar")
|
| 173 |
+
>>> list(d.items())
|
| 174 |
+
[
|
| 175 |
+
('A', '1, 2'),
|
| 176 |
+
('B', 'foo'),
|
| 177 |
+
('B', 'bar'),
|
| 178 |
+
]
|
| 179 |
+
|
| 180 |
+
This class conforms to the interface required by the MutableMapping ABC while
|
| 181 |
+
also giving us the nonstandard iteration behavior we want; items with duplicate
|
| 182 |
+
keys, ordered by time of first insertion.
|
| 183 |
+
"""
|
| 184 |
+
|
| 185 |
+
_headers: HTTPHeaderDict
|
| 186 |
+
|
| 187 |
+
def __init__(self, headers: HTTPHeaderDict) -> None:
|
| 188 |
+
self._headers = headers
|
| 189 |
+
|
| 190 |
+
def __len__(self) -> int:
|
| 191 |
+
return len(list(self._headers.iteritems()))
|
| 192 |
+
|
| 193 |
+
def __iter__(self) -> typing.Iterator[tuple[str, str]]:
|
| 194 |
+
return self._headers.iteritems()
|
| 195 |
+
|
| 196 |
+
def __contains__(self, item: object) -> bool:
|
| 197 |
+
if isinstance(item, tuple) and len(item) == 2:
|
| 198 |
+
passed_key, passed_val = item
|
| 199 |
+
if isinstance(passed_key, str) and isinstance(passed_val, str):
|
| 200 |
+
return self._headers._has_value_for_header(passed_key, passed_val)
|
| 201 |
+
return False
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
| 205 |
+
"""
|
| 206 |
+
:param headers:
|
| 207 |
+
An iterable of field-value pairs. Must not contain multiple field names
|
| 208 |
+
when compared case-insensitively.
|
| 209 |
+
|
| 210 |
+
:param kwargs:
|
| 211 |
+
Additional field-value pairs to pass in to ``dict.update``.
|
| 212 |
+
|
| 213 |
+
A ``dict`` like container for storing HTTP Headers.
|
| 214 |
+
|
| 215 |
+
Field names are stored and compared case-insensitively in compliance with
|
| 216 |
+
RFC 7230. Iteration provides the first case-sensitive key seen for each
|
| 217 |
+
case-insensitive pair.
|
| 218 |
+
|
| 219 |
+
Using ``__setitem__`` syntax overwrites fields that compare equal
|
| 220 |
+
case-insensitively in order to maintain ``dict``'s api. For fields that
|
| 221 |
+
compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add``
|
| 222 |
+
in a loop.
|
| 223 |
+
|
| 224 |
+
If multiple fields that are equal case-insensitively are passed to the
|
| 225 |
+
constructor or ``.update``, the behavior is undefined and some will be
|
| 226 |
+
lost.
|
| 227 |
+
|
| 228 |
+
>>> headers = HTTPHeaderDict()
|
| 229 |
+
>>> headers.add('Set-Cookie', 'foo=bar')
|
| 230 |
+
>>> headers.add('set-cookie', 'baz=quxx')
|
| 231 |
+
>>> headers['content-length'] = '7'
|
| 232 |
+
>>> headers['SET-cookie']
|
| 233 |
+
'foo=bar, baz=quxx'
|
| 234 |
+
>>> headers['Content-Length']
|
| 235 |
+
'7'
|
| 236 |
+
"""
|
| 237 |
+
|
| 238 |
+
_container: typing.MutableMapping[str, list[str]]
|
| 239 |
+
|
| 240 |
+
def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
|
| 241 |
+
super().__init__()
|
| 242 |
+
self._container = {} # 'dict' is insert-ordered
|
| 243 |
+
if headers is not None:
|
| 244 |
+
if isinstance(headers, HTTPHeaderDict):
|
| 245 |
+
self._copy_from(headers)
|
| 246 |
+
else:
|
| 247 |
+
self.extend(headers)
|
| 248 |
+
if kwargs:
|
| 249 |
+
self.extend(kwargs)
|
| 250 |
+
|
| 251 |
+
def __setitem__(self, key: str, val: str) -> None:
|
| 252 |
+
# avoid a bytes/str comparison by decoding before httplib
|
| 253 |
+
if isinstance(key, bytes):
|
| 254 |
+
key = key.decode("latin-1")
|
| 255 |
+
self._container[key.lower()] = [key, val]
|
| 256 |
+
|
| 257 |
+
def __getitem__(self, key: str) -> str:
|
| 258 |
+
if isinstance(key, bytes):
|
| 259 |
+
key = key.decode("latin-1")
|
| 260 |
+
val = self._container[key.lower()]
|
| 261 |
+
return ", ".join(val[1:])
|
| 262 |
+
|
| 263 |
+
def __delitem__(self, key: str) -> None:
|
| 264 |
+
if isinstance(key, bytes):
|
| 265 |
+
key = key.decode("latin-1")
|
| 266 |
+
del self._container[key.lower()]
|
| 267 |
+
|
| 268 |
+
def __contains__(self, key: object) -> bool:
|
| 269 |
+
if isinstance(key, bytes):
|
| 270 |
+
key = key.decode("latin-1")
|
| 271 |
+
if isinstance(key, str):
|
| 272 |
+
return key.lower() in self._container
|
| 273 |
+
return False
|
| 274 |
+
|
| 275 |
+
def setdefault(self, key: str, default: str = "") -> str:
|
| 276 |
+
return super().setdefault(key, default)
|
| 277 |
+
|
| 278 |
+
def __eq__(self, other: object) -> bool:
|
| 279 |
+
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
| 280 |
+
if maybe_constructable is None:
|
| 281 |
+
return False
|
| 282 |
+
else:
|
| 283 |
+
other_as_http_header_dict = type(self)(maybe_constructable)
|
| 284 |
+
|
| 285 |
+
return {k.lower(): v for k, v in self.itermerged()} == {
|
| 286 |
+
k.lower(): v for k, v in other_as_http_header_dict.itermerged()
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
def __ne__(self, other: object) -> bool:
|
| 290 |
+
return not self.__eq__(other)
|
| 291 |
+
|
| 292 |
+
def __len__(self) -> int:
|
| 293 |
+
return len(self._container)
|
| 294 |
+
|
| 295 |
+
def __iter__(self) -> typing.Iterator[str]:
|
| 296 |
+
# Only provide the originally cased names
|
| 297 |
+
for vals in self._container.values():
|
| 298 |
+
yield vals[0]
|
| 299 |
+
|
| 300 |
+
def discard(self, key: str) -> None:
|
| 301 |
+
try:
|
| 302 |
+
del self[key]
|
| 303 |
+
except KeyError:
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
def add(self, key: str, val: str, *, combine: bool = False) -> None:
|
| 307 |
+
"""Adds a (name, value) pair, doesn't overwrite the value if it already
|
| 308 |
+
exists.
|
| 309 |
+
|
| 310 |
+
If this is called with combine=True, instead of adding a new header value
|
| 311 |
+
as a distinct item during iteration, this will instead append the value to
|
| 312 |
+
any existing header value with a comma. If no existing header value exists
|
| 313 |
+
for the key, then the value will simply be added, ignoring the combine parameter.
|
| 314 |
+
|
| 315 |
+
>>> headers = HTTPHeaderDict(foo='bar')
|
| 316 |
+
>>> headers.add('Foo', 'baz')
|
| 317 |
+
>>> headers['foo']
|
| 318 |
+
'bar, baz'
|
| 319 |
+
>>> list(headers.items())
|
| 320 |
+
[('foo', 'bar'), ('foo', 'baz')]
|
| 321 |
+
>>> headers.add('foo', 'quz', combine=True)
|
| 322 |
+
>>> list(headers.items())
|
| 323 |
+
[('foo', 'bar, baz, quz')]
|
| 324 |
+
"""
|
| 325 |
+
# avoid a bytes/str comparison by decoding before httplib
|
| 326 |
+
if isinstance(key, bytes):
|
| 327 |
+
key = key.decode("latin-1")
|
| 328 |
+
key_lower = key.lower()
|
| 329 |
+
new_vals = [key, val]
|
| 330 |
+
# Keep the common case aka no item present as fast as possible
|
| 331 |
+
vals = self._container.setdefault(key_lower, new_vals)
|
| 332 |
+
if new_vals is not vals:
|
| 333 |
+
# if there are values here, then there is at least the initial
|
| 334 |
+
# key/value pair
|
| 335 |
+
assert len(vals) >= 2
|
| 336 |
+
if combine:
|
| 337 |
+
vals[-1] = vals[-1] + ", " + val
|
| 338 |
+
else:
|
| 339 |
+
vals.append(val)
|
| 340 |
+
|
| 341 |
+
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
|
| 342 |
+
"""Generic import function for any type of header-like object.
|
| 343 |
+
Adapted version of MutableMapping.update in order to insert items
|
| 344 |
+
with self.add instead of self.__setitem__
|
| 345 |
+
"""
|
| 346 |
+
if len(args) > 1:
|
| 347 |
+
raise TypeError(
|
| 348 |
+
f"extend() takes at most 1 positional arguments ({len(args)} given)"
|
| 349 |
+
)
|
| 350 |
+
other = args[0] if len(args) >= 1 else ()
|
| 351 |
+
|
| 352 |
+
if isinstance(other, HTTPHeaderDict):
|
| 353 |
+
for key, val in other.iteritems():
|
| 354 |
+
self.add(key, val)
|
| 355 |
+
elif isinstance(other, typing.Mapping):
|
| 356 |
+
for key, val in other.items():
|
| 357 |
+
self.add(key, val)
|
| 358 |
+
elif isinstance(other, typing.Iterable):
|
| 359 |
+
other = typing.cast(typing.Iterable[tuple[str, str]], other)
|
| 360 |
+
for key, value in other:
|
| 361 |
+
self.add(key, value)
|
| 362 |
+
elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
|
| 363 |
+
# THIS IS NOT A TYPESAFE BRANCH
|
| 364 |
+
# In this branch, the object has a `keys` attr but is not a Mapping or any of
|
| 365 |
+
# the other types indicated in the method signature. We do some stuff with
|
| 366 |
+
# it as though it partially implements the Mapping interface, but we're not
|
| 367 |
+
# doing that stuff safely AT ALL.
|
| 368 |
+
for key in other.keys():
|
| 369 |
+
self.add(key, other[key])
|
| 370 |
+
|
| 371 |
+
for key, value in kwargs.items():
|
| 372 |
+
self.add(key, value)
|
| 373 |
+
|
| 374 |
+
@typing.overload
|
| 375 |
+
def getlist(self, key: str) -> list[str]: ...
|
| 376 |
+
|
| 377 |
+
@typing.overload
|
| 378 |
+
def getlist(self, key: str, default: _DT) -> list[str] | _DT: ...
|
| 379 |
+
|
| 380 |
+
def getlist(
|
| 381 |
+
self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
|
| 382 |
+
) -> list[str] | _DT:
|
| 383 |
+
"""Returns a list of all the values for the named field. Returns an
|
| 384 |
+
empty list if the key doesn't exist."""
|
| 385 |
+
if isinstance(key, bytes):
|
| 386 |
+
key = key.decode("latin-1")
|
| 387 |
+
try:
|
| 388 |
+
vals = self._container[key.lower()]
|
| 389 |
+
except KeyError:
|
| 390 |
+
if default is _Sentinel.not_passed:
|
| 391 |
+
# _DT is unbound; empty list is instance of List[str]
|
| 392 |
+
return []
|
| 393 |
+
# _DT is bound; default is instance of _DT
|
| 394 |
+
return default
|
| 395 |
+
else:
|
| 396 |
+
# _DT may or may not be bound; vals[1:] is instance of List[str], which
|
| 397 |
+
# meets our external interface requirement of `Union[List[str], _DT]`.
|
| 398 |
+
return vals[1:]
|
| 399 |
+
|
| 400 |
+
def _prepare_for_method_change(self) -> Self:
|
| 401 |
+
"""
|
| 402 |
+
Remove content-specific header fields before changing the request
|
| 403 |
+
method to GET or HEAD according to RFC 9110, Section 15.4.
|
| 404 |
+
"""
|
| 405 |
+
content_specific_headers = [
|
| 406 |
+
"Content-Encoding",
|
| 407 |
+
"Content-Language",
|
| 408 |
+
"Content-Location",
|
| 409 |
+
"Content-Type",
|
| 410 |
+
"Content-Length",
|
| 411 |
+
"Digest",
|
| 412 |
+
"Last-Modified",
|
| 413 |
+
]
|
| 414 |
+
for header in content_specific_headers:
|
| 415 |
+
self.discard(header)
|
| 416 |
+
return self
|
| 417 |
+
|
| 418 |
+
# Backwards compatibility for httplib
|
| 419 |
+
getheaders = getlist
|
| 420 |
+
getallmatchingheaders = getlist
|
| 421 |
+
iget = getlist
|
| 422 |
+
|
| 423 |
+
# Backwards compatibility for http.cookiejar
|
| 424 |
+
get_all = getlist
|
| 425 |
+
|
| 426 |
+
def __repr__(self) -> str:
|
| 427 |
+
return f"{type(self).__name__}({dict(self.itermerged())})"
|
| 428 |
+
|
| 429 |
+
def _copy_from(self, other: HTTPHeaderDict) -> None:
|
| 430 |
+
for key in other:
|
| 431 |
+
val = other.getlist(key)
|
| 432 |
+
self._container[key.lower()] = [key, *val]
|
| 433 |
+
|
| 434 |
+
def copy(self) -> Self:
|
| 435 |
+
clone = type(self)()
|
| 436 |
+
clone._copy_from(self)
|
| 437 |
+
return clone
|
| 438 |
+
|
| 439 |
+
def iteritems(self) -> typing.Iterator[tuple[str, str]]:
|
| 440 |
+
"""Iterate over all header lines, including duplicate ones."""
|
| 441 |
+
for key in self:
|
| 442 |
+
vals = self._container[key.lower()]
|
| 443 |
+
for val in vals[1:]:
|
| 444 |
+
yield vals[0], val
|
| 445 |
+
|
| 446 |
+
def itermerged(self) -> typing.Iterator[tuple[str, str]]:
|
| 447 |
+
"""Iterate over all headers, merging duplicate ones together."""
|
| 448 |
+
for key in self:
|
| 449 |
+
val = self._container[key.lower()]
|
| 450 |
+
yield val[0], ", ".join(val[1:])
|
| 451 |
+
|
| 452 |
+
def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
|
| 453 |
+
return HTTPHeaderDictItemView(self)
|
| 454 |
+
|
| 455 |
+
def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
|
| 456 |
+
if header_name in self:
|
| 457 |
+
return potential_value in self._container[header_name.lower()][1:]
|
| 458 |
+
return False
|
| 459 |
+
|
| 460 |
+
def __ior__(self, other: object) -> HTTPHeaderDict:
|
| 461 |
+
# Supports extending a header dict in-place using operator |=
|
| 462 |
+
# combining items with add instead of __setitem__
|
| 463 |
+
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
| 464 |
+
if maybe_constructable is None:
|
| 465 |
+
return NotImplemented
|
| 466 |
+
self.extend(maybe_constructable)
|
| 467 |
+
return self
|
| 468 |
+
|
| 469 |
+
def __or__(self, other: object) -> Self:
|
| 470 |
+
# Supports merging header dicts using operator |
|
| 471 |
+
# combining items with add instead of __setitem__
|
| 472 |
+
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
| 473 |
+
if maybe_constructable is None:
|
| 474 |
+
return NotImplemented
|
| 475 |
+
result = self.copy()
|
| 476 |
+
result.extend(maybe_constructable)
|
| 477 |
+
return result
|
| 478 |
+
|
| 479 |
+
def __ror__(self, other: object) -> Self:
|
| 480 |
+
# Supports merging header dicts using operator | when other is on left side
|
| 481 |
+
# combining items with add instead of __setitem__
|
| 482 |
+
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
| 483 |
+
if maybe_constructable is None:
|
| 484 |
+
return NotImplemented
|
| 485 |
+
result = type(self)(maybe_constructable)
|
| 486 |
+
result.extend(self)
|
| 487 |
+
return result
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/_request_methods.py
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json as _json
|
| 4 |
+
import typing
|
| 5 |
+
from urllib.parse import urlencode
|
| 6 |
+
|
| 7 |
+
from ._base_connection import _TYPE_BODY
|
| 8 |
+
from ._collections import HTTPHeaderDict
|
| 9 |
+
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
| 10 |
+
from .response import BaseHTTPResponse
|
| 11 |
+
|
| 12 |
+
__all__ = ["RequestMethods"]
|
| 13 |
+
|
| 14 |
+
_TYPE_ENCODE_URL_FIELDS = typing.Union[
|
| 15 |
+
typing.Sequence[tuple[str, typing.Union[str, bytes]]],
|
| 16 |
+
typing.Mapping[str, typing.Union[str, bytes]],
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class RequestMethods:
|
| 21 |
+
"""
|
| 22 |
+
Convenience mixin for classes who implement a :meth:`urlopen` method, such
|
| 23 |
+
as :class:`urllib3.HTTPConnectionPool` and
|
| 24 |
+
:class:`urllib3.PoolManager`.
|
| 25 |
+
|
| 26 |
+
Provides behavior for making common types of HTTP request methods and
|
| 27 |
+
decides which type of request field encoding to use.
|
| 28 |
+
|
| 29 |
+
Specifically,
|
| 30 |
+
|
| 31 |
+
:meth:`.request_encode_url` is for sending requests whose fields are
|
| 32 |
+
encoded in the URL (such as GET, HEAD, DELETE).
|
| 33 |
+
|
| 34 |
+
:meth:`.request_encode_body` is for sending requests whose fields are
|
| 35 |
+
encoded in the *body* of the request using multipart or www-form-urlencoded
|
| 36 |
+
(such as for POST, PUT, PATCH).
|
| 37 |
+
|
| 38 |
+
:meth:`.request` is for making any kind of request, it will look up the
|
| 39 |
+
appropriate encoding format and use one of the above two methods to make
|
| 40 |
+
the request.
|
| 41 |
+
|
| 42 |
+
Initializer parameters:
|
| 43 |
+
|
| 44 |
+
:param headers:
|
| 45 |
+
Headers to include with all requests, unless other headers are given
|
| 46 |
+
explicitly.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
|
| 50 |
+
|
| 51 |
+
def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
|
| 52 |
+
self.headers = headers or {}
|
| 53 |
+
|
| 54 |
+
def urlopen(
|
| 55 |
+
self,
|
| 56 |
+
method: str,
|
| 57 |
+
url: str,
|
| 58 |
+
body: _TYPE_BODY | None = None,
|
| 59 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 60 |
+
encode_multipart: bool = True,
|
| 61 |
+
multipart_boundary: str | None = None,
|
| 62 |
+
**kw: typing.Any,
|
| 63 |
+
) -> BaseHTTPResponse: # Abstract
|
| 64 |
+
raise NotImplementedError(
|
| 65 |
+
"Classes extending RequestMethods must implement "
|
| 66 |
+
"their own ``urlopen`` method."
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
def request(
|
| 70 |
+
self,
|
| 71 |
+
method: str,
|
| 72 |
+
url: str,
|
| 73 |
+
body: _TYPE_BODY | None = None,
|
| 74 |
+
fields: _TYPE_FIELDS | None = None,
|
| 75 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 76 |
+
json: typing.Any | None = None,
|
| 77 |
+
**urlopen_kw: typing.Any,
|
| 78 |
+
) -> BaseHTTPResponse:
|
| 79 |
+
"""
|
| 80 |
+
Make a request using :meth:`urlopen` with the appropriate encoding of
|
| 81 |
+
``fields`` based on the ``method`` used.
|
| 82 |
+
|
| 83 |
+
This is a convenience method that requires the least amount of manual
|
| 84 |
+
effort. It can be used in most situations, while still having the
|
| 85 |
+
option to drop down to more specific methods when necessary, such as
|
| 86 |
+
:meth:`request_encode_url`, :meth:`request_encode_body`,
|
| 87 |
+
or even the lowest level :meth:`urlopen`.
|
| 88 |
+
|
| 89 |
+
:param method:
|
| 90 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 91 |
+
|
| 92 |
+
:param url:
|
| 93 |
+
The URL to perform the request on.
|
| 94 |
+
|
| 95 |
+
:param body:
|
| 96 |
+
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
| 97 |
+
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
| 98 |
+
|
| 99 |
+
:param fields:
|
| 100 |
+
Data to encode and send in the URL or request body, depending on ``method``.
|
| 101 |
+
|
| 102 |
+
:param headers:
|
| 103 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 104 |
+
If-None-Match, etc. If None, pool headers are used. If provided,
|
| 105 |
+
these headers completely replace any pool-specific headers.
|
| 106 |
+
|
| 107 |
+
:param json:
|
| 108 |
+
Data to encode and send as JSON with UTF-encoded in the request body.
|
| 109 |
+
The ``"Content-Type"`` header will be set to ``"application/json"``
|
| 110 |
+
unless specified otherwise.
|
| 111 |
+
"""
|
| 112 |
+
method = method.upper()
|
| 113 |
+
|
| 114 |
+
if json is not None and body is not None:
|
| 115 |
+
raise TypeError(
|
| 116 |
+
"request got values for both 'body' and 'json' parameters which are mutually exclusive"
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
if json is not None:
|
| 120 |
+
if headers is None:
|
| 121 |
+
headers = self.headers
|
| 122 |
+
|
| 123 |
+
if not ("content-type" in map(str.lower, headers.keys())):
|
| 124 |
+
headers = HTTPHeaderDict(headers)
|
| 125 |
+
headers["Content-Type"] = "application/json"
|
| 126 |
+
|
| 127 |
+
body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
|
| 128 |
+
"utf-8"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
if body is not None:
|
| 132 |
+
urlopen_kw["body"] = body
|
| 133 |
+
|
| 134 |
+
if method in self._encode_url_methods:
|
| 135 |
+
return self.request_encode_url(
|
| 136 |
+
method,
|
| 137 |
+
url,
|
| 138 |
+
fields=fields, # type: ignore[arg-type]
|
| 139 |
+
headers=headers,
|
| 140 |
+
**urlopen_kw,
|
| 141 |
+
)
|
| 142 |
+
else:
|
| 143 |
+
return self.request_encode_body(
|
| 144 |
+
method, url, fields=fields, headers=headers, **urlopen_kw
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
def request_encode_url(
|
| 148 |
+
self,
|
| 149 |
+
method: str,
|
| 150 |
+
url: str,
|
| 151 |
+
fields: _TYPE_ENCODE_URL_FIELDS | None = None,
|
| 152 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 153 |
+
**urlopen_kw: str,
|
| 154 |
+
) -> BaseHTTPResponse:
|
| 155 |
+
"""
|
| 156 |
+
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
| 157 |
+
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
|
| 158 |
+
|
| 159 |
+
:param method:
|
| 160 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 161 |
+
|
| 162 |
+
:param url:
|
| 163 |
+
The URL to perform the request on.
|
| 164 |
+
|
| 165 |
+
:param fields:
|
| 166 |
+
Data to encode and send in the URL.
|
| 167 |
+
|
| 168 |
+
:param headers:
|
| 169 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 170 |
+
If-None-Match, etc. If None, pool headers are used. If provided,
|
| 171 |
+
these headers completely replace any pool-specific headers.
|
| 172 |
+
"""
|
| 173 |
+
if headers is None:
|
| 174 |
+
headers = self.headers
|
| 175 |
+
|
| 176 |
+
extra_kw: dict[str, typing.Any] = {"headers": headers}
|
| 177 |
+
extra_kw.update(urlopen_kw)
|
| 178 |
+
|
| 179 |
+
if fields:
|
| 180 |
+
url += "?" + urlencode(fields)
|
| 181 |
+
|
| 182 |
+
return self.urlopen(method, url, **extra_kw)
|
| 183 |
+
|
| 184 |
+
def request_encode_body(
|
| 185 |
+
self,
|
| 186 |
+
method: str,
|
| 187 |
+
url: str,
|
| 188 |
+
fields: _TYPE_FIELDS | None = None,
|
| 189 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 190 |
+
encode_multipart: bool = True,
|
| 191 |
+
multipart_boundary: str | None = None,
|
| 192 |
+
**urlopen_kw: str,
|
| 193 |
+
) -> BaseHTTPResponse:
|
| 194 |
+
"""
|
| 195 |
+
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
| 196 |
+
the body. This is useful for request methods like POST, PUT, PATCH, etc.
|
| 197 |
+
|
| 198 |
+
When ``encode_multipart=True`` (default), then
|
| 199 |
+
:func:`urllib3.encode_multipart_formdata` is used to encode
|
| 200 |
+
the payload with the appropriate content type. Otherwise
|
| 201 |
+
:func:`urllib.parse.urlencode` is used with the
|
| 202 |
+
'application/x-www-form-urlencoded' content type.
|
| 203 |
+
|
| 204 |
+
Multipart encoding must be used when posting files, and it's reasonably
|
| 205 |
+
safe to use it in other times too. However, it may break request
|
| 206 |
+
signing, such as with OAuth.
|
| 207 |
+
|
| 208 |
+
Supports an optional ``fields`` parameter of key/value strings AND
|
| 209 |
+
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
|
| 210 |
+
the MIME type is optional. For example::
|
| 211 |
+
|
| 212 |
+
fields = {
|
| 213 |
+
'foo': 'bar',
|
| 214 |
+
'fakefile': ('foofile.txt', 'contents of foofile'),
|
| 215 |
+
'realfile': ('barfile.txt', open('realfile').read()),
|
| 216 |
+
'typedfile': ('bazfile.bin', open('bazfile').read(),
|
| 217 |
+
'image/jpeg'),
|
| 218 |
+
'nonamefile': 'contents of nonamefile field',
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
When uploading a file, providing a filename (the first parameter of the
|
| 222 |
+
tuple) is optional but recommended to best mimic behavior of browsers.
|
| 223 |
+
|
| 224 |
+
Note that if ``headers`` are supplied, the 'Content-Type' header will
|
| 225 |
+
be overwritten because it depends on the dynamic random boundary string
|
| 226 |
+
which is used to compose the body of the request. The random boundary
|
| 227 |
+
string can be explicitly set with the ``multipart_boundary`` parameter.
|
| 228 |
+
|
| 229 |
+
:param method:
|
| 230 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 231 |
+
|
| 232 |
+
:param url:
|
| 233 |
+
The URL to perform the request on.
|
| 234 |
+
|
| 235 |
+
:param fields:
|
| 236 |
+
Data to encode and send in the request body.
|
| 237 |
+
|
| 238 |
+
:param headers:
|
| 239 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 240 |
+
If-None-Match, etc. If None, pool headers are used. If provided,
|
| 241 |
+
these headers completely replace any pool-specific headers.
|
| 242 |
+
|
| 243 |
+
:param encode_multipart:
|
| 244 |
+
If True, encode the ``fields`` using the multipart/form-data MIME
|
| 245 |
+
format.
|
| 246 |
+
|
| 247 |
+
:param multipart_boundary:
|
| 248 |
+
If not specified, then a random boundary will be generated using
|
| 249 |
+
:func:`urllib3.filepost.choose_boundary`.
|
| 250 |
+
"""
|
| 251 |
+
if headers is None:
|
| 252 |
+
headers = self.headers
|
| 253 |
+
|
| 254 |
+
extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
|
| 255 |
+
body: bytes | str
|
| 256 |
+
|
| 257 |
+
if fields:
|
| 258 |
+
if "body" in urlopen_kw:
|
| 259 |
+
raise TypeError(
|
| 260 |
+
"request got values for both 'fields' and 'body', can only specify one."
|
| 261 |
+
)
|
| 262 |
+
|
| 263 |
+
if encode_multipart:
|
| 264 |
+
body, content_type = encode_multipart_formdata(
|
| 265 |
+
fields, boundary=multipart_boundary
|
| 266 |
+
)
|
| 267 |
+
else:
|
| 268 |
+
body, content_type = (
|
| 269 |
+
urlencode(fields), # type: ignore[arg-type]
|
| 270 |
+
"application/x-www-form-urlencoded",
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
extra_kw["body"] = body
|
| 274 |
+
extra_kw["headers"].setdefault("Content-Type", content_type)
|
| 275 |
+
|
| 276 |
+
extra_kw.update(urlopen_kw)
|
| 277 |
+
|
| 278 |
+
return self.urlopen(method, url, **extra_kw)
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/_version.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# file generated by setuptools-scm
|
| 2 |
+
# don't change, don't track in version control
|
| 3 |
+
|
| 4 |
+
__all__ = [
|
| 5 |
+
"__version__",
|
| 6 |
+
"__version_tuple__",
|
| 7 |
+
"version",
|
| 8 |
+
"version_tuple",
|
| 9 |
+
"__commit_id__",
|
| 10 |
+
"commit_id",
|
| 11 |
+
]
|
| 12 |
+
|
| 13 |
+
TYPE_CHECKING = False
|
| 14 |
+
if TYPE_CHECKING:
|
| 15 |
+
from typing import Tuple
|
| 16 |
+
from typing import Union
|
| 17 |
+
|
| 18 |
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
| 19 |
+
COMMIT_ID = Union[str, None]
|
| 20 |
+
else:
|
| 21 |
+
VERSION_TUPLE = object
|
| 22 |
+
COMMIT_ID = object
|
| 23 |
+
|
| 24 |
+
version: str
|
| 25 |
+
__version__: str
|
| 26 |
+
__version_tuple__: VERSION_TUPLE
|
| 27 |
+
version_tuple: VERSION_TUPLE
|
| 28 |
+
commit_id: COMMIT_ID
|
| 29 |
+
__commit_id__: COMMIT_ID
|
| 30 |
+
|
| 31 |
+
__version__ = version = '2.6.3'
|
| 32 |
+
__version_tuple__ = version_tuple = (2, 6, 3)
|
| 33 |
+
|
| 34 |
+
__commit_id__ = commit_id = None
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/connection.py
ADDED
|
@@ -0,0 +1,1099 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import datetime
|
| 4 |
+
import http.client
|
| 5 |
+
import logging
|
| 6 |
+
import os
|
| 7 |
+
import re
|
| 8 |
+
import socket
|
| 9 |
+
import sys
|
| 10 |
+
import threading
|
| 11 |
+
import typing
|
| 12 |
+
import warnings
|
| 13 |
+
from http.client import HTTPConnection as _HTTPConnection
|
| 14 |
+
from http.client import HTTPException as HTTPException # noqa: F401
|
| 15 |
+
from http.client import ResponseNotReady
|
| 16 |
+
from socket import timeout as SocketTimeout
|
| 17 |
+
|
| 18 |
+
if typing.TYPE_CHECKING:
|
| 19 |
+
from .response import HTTPResponse
|
| 20 |
+
from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
|
| 21 |
+
from .util.ssltransport import SSLTransport
|
| 22 |
+
|
| 23 |
+
from ._collections import HTTPHeaderDict
|
| 24 |
+
from .http2 import probe as http2_probe
|
| 25 |
+
from .util.response import assert_header_parsing
|
| 26 |
+
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
|
| 27 |
+
from .util.util import to_str
|
| 28 |
+
from .util.wait import wait_for_read
|
| 29 |
+
|
| 30 |
+
try: # Compiled with SSL?
|
| 31 |
+
import ssl
|
| 32 |
+
|
| 33 |
+
BaseSSLError = ssl.SSLError
|
| 34 |
+
except (ImportError, AttributeError):
|
| 35 |
+
ssl = None # type: ignore[assignment]
|
| 36 |
+
|
| 37 |
+
class BaseSSLError(BaseException): # type: ignore[no-redef]
|
| 38 |
+
pass
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
from ._base_connection import _TYPE_BODY
|
| 42 |
+
from ._base_connection import ProxyConfig as ProxyConfig
|
| 43 |
+
from ._base_connection import _ResponseOptions as _ResponseOptions
|
| 44 |
+
from ._version import __version__
|
| 45 |
+
from .exceptions import (
|
| 46 |
+
ConnectTimeoutError,
|
| 47 |
+
HeaderParsingError,
|
| 48 |
+
NameResolutionError,
|
| 49 |
+
NewConnectionError,
|
| 50 |
+
ProxyError,
|
| 51 |
+
SystemTimeWarning,
|
| 52 |
+
)
|
| 53 |
+
from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
|
| 54 |
+
from .util.request import body_to_chunks
|
| 55 |
+
from .util.ssl_ import assert_fingerprint as _assert_fingerprint
|
| 56 |
+
from .util.ssl_ import (
|
| 57 |
+
create_urllib3_context,
|
| 58 |
+
is_ipaddress,
|
| 59 |
+
resolve_cert_reqs,
|
| 60 |
+
resolve_ssl_version,
|
| 61 |
+
ssl_wrap_socket,
|
| 62 |
+
)
|
| 63 |
+
from .util.ssl_match_hostname import CertificateError, match_hostname
|
| 64 |
+
from .util.url import Url
|
| 65 |
+
|
| 66 |
+
# Not a no-op, we're adding this to the namespace so it can be imported.
|
| 67 |
+
ConnectionError = ConnectionError
|
| 68 |
+
BrokenPipeError = BrokenPipeError
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
log = logging.getLogger(__name__)
|
| 72 |
+
|
| 73 |
+
port_by_scheme = {"http": 80, "https": 443}
|
| 74 |
+
|
| 75 |
+
# When it comes time to update this value as a part of regular maintenance
|
| 76 |
+
# (ie test_recent_date is failing) update it to ~6 months before the current date.
|
| 77 |
+
RECENT_DATE = datetime.date(2025, 1, 1)
|
| 78 |
+
|
| 79 |
+
_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class HTTPConnection(_HTTPConnection):
|
| 83 |
+
"""
|
| 84 |
+
Based on :class:`http.client.HTTPConnection` but provides an extra constructor
|
| 85 |
+
backwards-compatibility layer between older and newer Pythons.
|
| 86 |
+
|
| 87 |
+
Additional keyword parameters are used to configure attributes of the connection.
|
| 88 |
+
Accepted parameters include:
|
| 89 |
+
|
| 90 |
+
- ``source_address``: Set the source address for the current connection.
|
| 91 |
+
- ``socket_options``: Set specific options on the underlying socket. If not specified, then
|
| 92 |
+
defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
|
| 93 |
+
Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
|
| 94 |
+
|
| 95 |
+
For example, if you wish to enable TCP Keep Alive in addition to the defaults,
|
| 96 |
+
you might pass:
|
| 97 |
+
|
| 98 |
+
.. code-block:: python
|
| 99 |
+
|
| 100 |
+
HTTPConnection.default_socket_options + [
|
| 101 |
+
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
|
| 102 |
+
]
|
| 103 |
+
|
| 104 |
+
Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
|
| 105 |
+
"""
|
| 106 |
+
|
| 107 |
+
default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
|
| 108 |
+
|
| 109 |
+
#: Disable Nagle's algorithm by default.
|
| 110 |
+
#: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
|
| 111 |
+
default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
|
| 112 |
+
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
| 113 |
+
]
|
| 114 |
+
|
| 115 |
+
#: Whether this connection verifies the host's certificate.
|
| 116 |
+
is_verified: bool = False
|
| 117 |
+
|
| 118 |
+
#: Whether this proxy connection verified the proxy host's certificate.
|
| 119 |
+
# If no proxy is currently connected to the value will be ``None``.
|
| 120 |
+
proxy_is_verified: bool | None = None
|
| 121 |
+
|
| 122 |
+
blocksize: int
|
| 123 |
+
source_address: tuple[str, int] | None
|
| 124 |
+
socket_options: connection._TYPE_SOCKET_OPTIONS | None
|
| 125 |
+
|
| 126 |
+
_has_connected_to_proxy: bool
|
| 127 |
+
_response_options: _ResponseOptions | None
|
| 128 |
+
_tunnel_host: str | None
|
| 129 |
+
_tunnel_port: int | None
|
| 130 |
+
_tunnel_scheme: str | None
|
| 131 |
+
|
| 132 |
+
def __init__(
|
| 133 |
+
self,
|
| 134 |
+
host: str,
|
| 135 |
+
port: int | None = None,
|
| 136 |
+
*,
|
| 137 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 138 |
+
source_address: tuple[str, int] | None = None,
|
| 139 |
+
blocksize: int = 16384,
|
| 140 |
+
socket_options: None | (
|
| 141 |
+
connection._TYPE_SOCKET_OPTIONS
|
| 142 |
+
) = default_socket_options,
|
| 143 |
+
proxy: Url | None = None,
|
| 144 |
+
proxy_config: ProxyConfig | None = None,
|
| 145 |
+
) -> None:
|
| 146 |
+
super().__init__(
|
| 147 |
+
host=host,
|
| 148 |
+
port=port,
|
| 149 |
+
timeout=Timeout.resolve_default_timeout(timeout),
|
| 150 |
+
source_address=source_address,
|
| 151 |
+
blocksize=blocksize,
|
| 152 |
+
)
|
| 153 |
+
self.socket_options = socket_options
|
| 154 |
+
self.proxy = proxy
|
| 155 |
+
self.proxy_config = proxy_config
|
| 156 |
+
|
| 157 |
+
self._has_connected_to_proxy = False
|
| 158 |
+
self._response_options = None
|
| 159 |
+
self._tunnel_host: str | None = None
|
| 160 |
+
self._tunnel_port: int | None = None
|
| 161 |
+
self._tunnel_scheme: str | None = None
|
| 162 |
+
|
| 163 |
+
def __str__(self) -> str:
|
| 164 |
+
return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
|
| 165 |
+
|
| 166 |
+
def __repr__(self) -> str:
|
| 167 |
+
return f"<{self} at {id(self):#x}>"
|
| 168 |
+
|
| 169 |
+
@property
|
| 170 |
+
def host(self) -> str:
|
| 171 |
+
"""
|
| 172 |
+
Getter method to remove any trailing dots that indicate the hostname is an FQDN.
|
| 173 |
+
|
| 174 |
+
In general, SSL certificates don't include the trailing dot indicating a
|
| 175 |
+
fully-qualified domain name, and thus, they don't validate properly when
|
| 176 |
+
checked against a domain name that includes the dot. In addition, some
|
| 177 |
+
servers may not expect to receive the trailing dot when provided.
|
| 178 |
+
|
| 179 |
+
However, the hostname with trailing dot is critical to DNS resolution; doing a
|
| 180 |
+
lookup with the trailing dot will properly only resolve the appropriate FQDN,
|
| 181 |
+
whereas a lookup without a trailing dot will search the system's search domain
|
| 182 |
+
list. Thus, it's important to keep the original host around for use only in
|
| 183 |
+
those cases where it's appropriate (i.e., when doing DNS lookup to establish the
|
| 184 |
+
actual TCP connection across which we're going to send HTTP requests).
|
| 185 |
+
"""
|
| 186 |
+
return self._dns_host.rstrip(".")
|
| 187 |
+
|
| 188 |
+
@host.setter
|
| 189 |
+
def host(self, value: str) -> None:
|
| 190 |
+
"""
|
| 191 |
+
Setter for the `host` property.
|
| 192 |
+
|
| 193 |
+
We assume that only urllib3 uses the _dns_host attribute; httplib itself
|
| 194 |
+
only uses `host`, and it seems reasonable that other libraries follow suit.
|
| 195 |
+
"""
|
| 196 |
+
self._dns_host = value
|
| 197 |
+
|
| 198 |
+
def _new_conn(self) -> socket.socket:
|
| 199 |
+
"""Establish a socket connection and set nodelay settings on it.
|
| 200 |
+
|
| 201 |
+
:return: New socket connection.
|
| 202 |
+
"""
|
| 203 |
+
try:
|
| 204 |
+
sock = connection.create_connection(
|
| 205 |
+
(self._dns_host, self.port),
|
| 206 |
+
self.timeout,
|
| 207 |
+
source_address=self.source_address,
|
| 208 |
+
socket_options=self.socket_options,
|
| 209 |
+
)
|
| 210 |
+
except socket.gaierror as e:
|
| 211 |
+
raise NameResolutionError(self.host, self, e) from e
|
| 212 |
+
except SocketTimeout as e:
|
| 213 |
+
raise ConnectTimeoutError(
|
| 214 |
+
self,
|
| 215 |
+
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
| 216 |
+
) from e
|
| 217 |
+
|
| 218 |
+
except OSError as e:
|
| 219 |
+
raise NewConnectionError(
|
| 220 |
+
self, f"Failed to establish a new connection: {e}"
|
| 221 |
+
) from e
|
| 222 |
+
|
| 223 |
+
sys.audit("http.client.connect", self, self.host, self.port)
|
| 224 |
+
|
| 225 |
+
return sock
|
| 226 |
+
|
| 227 |
+
def set_tunnel(
|
| 228 |
+
self,
|
| 229 |
+
host: str,
|
| 230 |
+
port: int | None = None,
|
| 231 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 232 |
+
scheme: str = "http",
|
| 233 |
+
) -> None:
|
| 234 |
+
if scheme not in ("http", "https"):
|
| 235 |
+
raise ValueError(
|
| 236 |
+
f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
|
| 237 |
+
)
|
| 238 |
+
super().set_tunnel(host, port=port, headers=headers)
|
| 239 |
+
self._tunnel_scheme = scheme
|
| 240 |
+
|
| 241 |
+
if sys.version_info < (3, 11, 9) or ((3, 12) <= sys.version_info < (3, 12, 3)):
|
| 242 |
+
# Taken from python/cpython#100986 which was backported in 3.11.9 and 3.12.3.
|
| 243 |
+
# When using connection_from_host, host will come without brackets.
|
| 244 |
+
def _wrap_ipv6(self, ip: bytes) -> bytes:
|
| 245 |
+
if b":" in ip and ip[0] != b"["[0]:
|
| 246 |
+
return b"[" + ip + b"]"
|
| 247 |
+
return ip
|
| 248 |
+
|
| 249 |
+
if sys.version_info < (3, 11, 9):
|
| 250 |
+
# `_tunnel` copied from 3.11.13 backporting
|
| 251 |
+
# https://github.com/python/cpython/commit/0d4026432591d43185568dd31cef6a034c4b9261
|
| 252 |
+
# and https://github.com/python/cpython/commit/6fbc61070fda2ffb8889e77e3b24bca4249ab4d1
|
| 253 |
+
def _tunnel(self) -> None:
|
| 254 |
+
_MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
|
| 255 |
+
connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
|
| 256 |
+
self._wrap_ipv6(self._tunnel_host.encode("ascii")), # type: ignore[union-attr]
|
| 257 |
+
self._tunnel_port,
|
| 258 |
+
)
|
| 259 |
+
headers = [connect]
|
| 260 |
+
for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
|
| 261 |
+
headers.append(f"{header}: {value}\r\n".encode("latin-1"))
|
| 262 |
+
headers.append(b"\r\n")
|
| 263 |
+
# Making a single send() call instead of one per line encourages
|
| 264 |
+
# the host OS to use a more optimal packet size instead of
|
| 265 |
+
# potentially emitting a series of small packets.
|
| 266 |
+
self.send(b"".join(headers))
|
| 267 |
+
del headers
|
| 268 |
+
|
| 269 |
+
response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
|
| 270 |
+
try:
|
| 271 |
+
(version, code, message) = response._read_status() # type: ignore[attr-defined]
|
| 272 |
+
|
| 273 |
+
if code != http.HTTPStatus.OK:
|
| 274 |
+
self.close()
|
| 275 |
+
raise OSError(
|
| 276 |
+
f"Tunnel connection failed: {code} {message.strip()}"
|
| 277 |
+
)
|
| 278 |
+
while True:
|
| 279 |
+
line = response.fp.readline(_MAXLINE + 1)
|
| 280 |
+
if len(line) > _MAXLINE:
|
| 281 |
+
raise http.client.LineTooLong("header line")
|
| 282 |
+
if not line:
|
| 283 |
+
# for sites which EOF without sending a trailer
|
| 284 |
+
break
|
| 285 |
+
if line in (b"\r\n", b"\n", b""):
|
| 286 |
+
break
|
| 287 |
+
|
| 288 |
+
if self.debuglevel > 0:
|
| 289 |
+
print("header:", line.decode())
|
| 290 |
+
finally:
|
| 291 |
+
response.close()
|
| 292 |
+
|
| 293 |
+
elif (3, 12) <= sys.version_info < (3, 12, 3):
|
| 294 |
+
# `_tunnel` copied from 3.12.11 backporting
|
| 295 |
+
# https://github.com/python/cpython/commit/23aef575c7629abcd4aaf028ebd226fb41a4b3c8
|
| 296 |
+
def _tunnel(self) -> None: # noqa: F811
|
| 297 |
+
connect = b"CONNECT %s:%d HTTP/1.1\r\n" % ( # type: ignore[str-format]
|
| 298 |
+
self._wrap_ipv6(self._tunnel_host.encode("idna")), # type: ignore[union-attr]
|
| 299 |
+
self._tunnel_port,
|
| 300 |
+
)
|
| 301 |
+
headers = [connect]
|
| 302 |
+
for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
|
| 303 |
+
headers.append(f"{header}: {value}\r\n".encode("latin-1"))
|
| 304 |
+
headers.append(b"\r\n")
|
| 305 |
+
# Making a single send() call instead of one per line encourages
|
| 306 |
+
# the host OS to use a more optimal packet size instead of
|
| 307 |
+
# potentially emitting a series of small packets.
|
| 308 |
+
self.send(b"".join(headers))
|
| 309 |
+
del headers
|
| 310 |
+
|
| 311 |
+
response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
|
| 312 |
+
try:
|
| 313 |
+
(version, code, message) = response._read_status() # type: ignore[attr-defined]
|
| 314 |
+
|
| 315 |
+
self._raw_proxy_headers = http.client._read_headers(response.fp) # type: ignore[attr-defined]
|
| 316 |
+
|
| 317 |
+
if self.debuglevel > 0:
|
| 318 |
+
for header in self._raw_proxy_headers:
|
| 319 |
+
print("header:", header.decode())
|
| 320 |
+
|
| 321 |
+
if code != http.HTTPStatus.OK:
|
| 322 |
+
self.close()
|
| 323 |
+
raise OSError(
|
| 324 |
+
f"Tunnel connection failed: {code} {message.strip()}"
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
finally:
|
| 328 |
+
response.close()
|
| 329 |
+
|
| 330 |
+
def connect(self) -> None:
|
| 331 |
+
self.sock = self._new_conn()
|
| 332 |
+
if self._tunnel_host:
|
| 333 |
+
# If we're tunneling it means we're connected to our proxy.
|
| 334 |
+
self._has_connected_to_proxy = True
|
| 335 |
+
|
| 336 |
+
# TODO: Fix tunnel so it doesn't depend on self.sock state.
|
| 337 |
+
self._tunnel()
|
| 338 |
+
|
| 339 |
+
# If there's a proxy to be connected to we are fully connected.
|
| 340 |
+
# This is set twice (once above and here) due to forwarding proxies
|
| 341 |
+
# not using tunnelling.
|
| 342 |
+
self._has_connected_to_proxy = bool(self.proxy)
|
| 343 |
+
|
| 344 |
+
if self._has_connected_to_proxy:
|
| 345 |
+
self.proxy_is_verified = False
|
| 346 |
+
|
| 347 |
+
@property
|
| 348 |
+
def is_closed(self) -> bool:
|
| 349 |
+
return self.sock is None
|
| 350 |
+
|
| 351 |
+
@property
|
| 352 |
+
def is_connected(self) -> bool:
|
| 353 |
+
if self.sock is None:
|
| 354 |
+
return False
|
| 355 |
+
return not wait_for_read(self.sock, timeout=0.0)
|
| 356 |
+
|
| 357 |
+
@property
|
| 358 |
+
def has_connected_to_proxy(self) -> bool:
|
| 359 |
+
return self._has_connected_to_proxy
|
| 360 |
+
|
| 361 |
+
@property
|
| 362 |
+
def proxy_is_forwarding(self) -> bool:
|
| 363 |
+
"""
|
| 364 |
+
Return True if a forwarding proxy is configured, else return False
|
| 365 |
+
"""
|
| 366 |
+
return bool(self.proxy) and self._tunnel_host is None
|
| 367 |
+
|
| 368 |
+
@property
|
| 369 |
+
def proxy_is_tunneling(self) -> bool:
|
| 370 |
+
"""
|
| 371 |
+
Return True if a tunneling proxy is configured, else return False
|
| 372 |
+
"""
|
| 373 |
+
return self._tunnel_host is not None
|
| 374 |
+
|
| 375 |
+
def close(self) -> None:
|
| 376 |
+
try:
|
| 377 |
+
super().close()
|
| 378 |
+
finally:
|
| 379 |
+
# Reset all stateful properties so connection
|
| 380 |
+
# can be re-used without leaking prior configs.
|
| 381 |
+
self.sock = None
|
| 382 |
+
self.is_verified = False
|
| 383 |
+
self.proxy_is_verified = None
|
| 384 |
+
self._has_connected_to_proxy = False
|
| 385 |
+
self._response_options = None
|
| 386 |
+
self._tunnel_host = None
|
| 387 |
+
self._tunnel_port = None
|
| 388 |
+
self._tunnel_scheme = None
|
| 389 |
+
|
| 390 |
+
def putrequest(
|
| 391 |
+
self,
|
| 392 |
+
method: str,
|
| 393 |
+
url: str,
|
| 394 |
+
skip_host: bool = False,
|
| 395 |
+
skip_accept_encoding: bool = False,
|
| 396 |
+
) -> None:
|
| 397 |
+
""""""
|
| 398 |
+
# Empty docstring because the indentation of CPython's implementation
|
| 399 |
+
# is broken but we don't want this method in our documentation.
|
| 400 |
+
match = _CONTAINS_CONTROL_CHAR_RE.search(method)
|
| 401 |
+
if match:
|
| 402 |
+
raise ValueError(
|
| 403 |
+
f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
return super().putrequest(
|
| 407 |
+
method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
|
| 411 |
+
""""""
|
| 412 |
+
if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
|
| 413 |
+
super().putheader(header, *values)
|
| 414 |
+
elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
|
| 415 |
+
skippable_headers = "', '".join(
|
| 416 |
+
[str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
|
| 417 |
+
)
|
| 418 |
+
raise ValueError(
|
| 419 |
+
f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
|
| 420 |
+
)
|
| 421 |
+
|
| 422 |
+
# `request` method's signature intentionally violates LSP.
|
| 423 |
+
# urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
|
| 424 |
+
def request( # type: ignore[override]
|
| 425 |
+
self,
|
| 426 |
+
method: str,
|
| 427 |
+
url: str,
|
| 428 |
+
body: _TYPE_BODY | None = None,
|
| 429 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 430 |
+
*,
|
| 431 |
+
chunked: bool = False,
|
| 432 |
+
preload_content: bool = True,
|
| 433 |
+
decode_content: bool = True,
|
| 434 |
+
enforce_content_length: bool = True,
|
| 435 |
+
) -> None:
|
| 436 |
+
# Update the inner socket's timeout value to send the request.
|
| 437 |
+
# This only triggers if the connection is re-used.
|
| 438 |
+
if self.sock is not None:
|
| 439 |
+
self.sock.settimeout(self.timeout)
|
| 440 |
+
|
| 441 |
+
# Store these values to be fed into the HTTPResponse
|
| 442 |
+
# object later. TODO: Remove this in favor of a real
|
| 443 |
+
# HTTP lifecycle mechanism.
|
| 444 |
+
|
| 445 |
+
# We have to store these before we call .request()
|
| 446 |
+
# because sometimes we can still salvage a response
|
| 447 |
+
# off the wire even if we aren't able to completely
|
| 448 |
+
# send the request body.
|
| 449 |
+
self._response_options = _ResponseOptions(
|
| 450 |
+
request_method=method,
|
| 451 |
+
request_url=url,
|
| 452 |
+
preload_content=preload_content,
|
| 453 |
+
decode_content=decode_content,
|
| 454 |
+
enforce_content_length=enforce_content_length,
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
if headers is None:
|
| 458 |
+
headers = {}
|
| 459 |
+
header_keys = frozenset(to_str(k.lower()) for k in headers)
|
| 460 |
+
skip_accept_encoding = "accept-encoding" in header_keys
|
| 461 |
+
skip_host = "host" in header_keys
|
| 462 |
+
self.putrequest(
|
| 463 |
+
method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
|
| 464 |
+
)
|
| 465 |
+
|
| 466 |
+
# Transform the body into an iterable of sendall()-able chunks
|
| 467 |
+
# and detect if an explicit Content-Length is doable.
|
| 468 |
+
chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
|
| 469 |
+
chunks = chunks_and_cl.chunks
|
| 470 |
+
content_length = chunks_and_cl.content_length
|
| 471 |
+
|
| 472 |
+
# When chunked is explicit set to 'True' we respect that.
|
| 473 |
+
if chunked:
|
| 474 |
+
if "transfer-encoding" not in header_keys:
|
| 475 |
+
self.putheader("Transfer-Encoding", "chunked")
|
| 476 |
+
else:
|
| 477 |
+
# Detect whether a framing mechanism is already in use. If so
|
| 478 |
+
# we respect that value, otherwise we pick chunked vs content-length
|
| 479 |
+
# depending on the type of 'body'.
|
| 480 |
+
if "content-length" in header_keys:
|
| 481 |
+
chunked = False
|
| 482 |
+
elif "transfer-encoding" in header_keys:
|
| 483 |
+
chunked = True
|
| 484 |
+
|
| 485 |
+
# Otherwise we go off the recommendation of 'body_to_chunks()'.
|
| 486 |
+
else:
|
| 487 |
+
chunked = False
|
| 488 |
+
if content_length is None:
|
| 489 |
+
if chunks is not None:
|
| 490 |
+
chunked = True
|
| 491 |
+
self.putheader("Transfer-Encoding", "chunked")
|
| 492 |
+
else:
|
| 493 |
+
self.putheader("Content-Length", str(content_length))
|
| 494 |
+
|
| 495 |
+
# Now that framing headers are out of the way we send all the other headers.
|
| 496 |
+
if "user-agent" not in header_keys:
|
| 497 |
+
self.putheader("User-Agent", _get_default_user_agent())
|
| 498 |
+
for header, value in headers.items():
|
| 499 |
+
self.putheader(header, value)
|
| 500 |
+
self.endheaders()
|
| 501 |
+
|
| 502 |
+
# If we're given a body we start sending that in chunks.
|
| 503 |
+
if chunks is not None:
|
| 504 |
+
for chunk in chunks:
|
| 505 |
+
# Sending empty chunks isn't allowed for TE: chunked
|
| 506 |
+
# as it indicates the end of the body.
|
| 507 |
+
if not chunk:
|
| 508 |
+
continue
|
| 509 |
+
if isinstance(chunk, str):
|
| 510 |
+
chunk = chunk.encode("utf-8")
|
| 511 |
+
if chunked:
|
| 512 |
+
self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
|
| 513 |
+
else:
|
| 514 |
+
self.send(chunk)
|
| 515 |
+
|
| 516 |
+
# Regardless of whether we have a body or not, if we're in
|
| 517 |
+
# chunked mode we want to send an explicit empty chunk.
|
| 518 |
+
if chunked:
|
| 519 |
+
self.send(b"0\r\n\r\n")
|
| 520 |
+
|
| 521 |
+
def request_chunked(
|
| 522 |
+
self,
|
| 523 |
+
method: str,
|
| 524 |
+
url: str,
|
| 525 |
+
body: _TYPE_BODY | None = None,
|
| 526 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 527 |
+
) -> None:
|
| 528 |
+
"""
|
| 529 |
+
Alternative to the common request method, which sends the
|
| 530 |
+
body with chunked encoding and not as one block
|
| 531 |
+
"""
|
| 532 |
+
warnings.warn(
|
| 533 |
+
"HTTPConnection.request_chunked() is deprecated and will be removed "
|
| 534 |
+
"in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
|
| 535 |
+
category=DeprecationWarning,
|
| 536 |
+
stacklevel=2,
|
| 537 |
+
)
|
| 538 |
+
self.request(method, url, body=body, headers=headers, chunked=True)
|
| 539 |
+
|
| 540 |
+
def getresponse( # type: ignore[override]
|
| 541 |
+
self,
|
| 542 |
+
) -> HTTPResponse:
|
| 543 |
+
"""
|
| 544 |
+
Get the response from the server.
|
| 545 |
+
|
| 546 |
+
If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
|
| 547 |
+
|
| 548 |
+
If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
|
| 549 |
+
"""
|
| 550 |
+
# Raise the same error as http.client.HTTPConnection
|
| 551 |
+
if self._response_options is None:
|
| 552 |
+
raise ResponseNotReady()
|
| 553 |
+
|
| 554 |
+
# Reset this attribute for being used again.
|
| 555 |
+
resp_options = self._response_options
|
| 556 |
+
self._response_options = None
|
| 557 |
+
|
| 558 |
+
# Since the connection's timeout value may have been updated
|
| 559 |
+
# we need to set the timeout on the socket.
|
| 560 |
+
self.sock.settimeout(self.timeout)
|
| 561 |
+
|
| 562 |
+
# This is needed here to avoid circular import errors
|
| 563 |
+
from .response import HTTPResponse
|
| 564 |
+
|
| 565 |
+
# Save a reference to the shutdown function before ownership is passed
|
| 566 |
+
# to httplib_response
|
| 567 |
+
# TODO should we implement it everywhere?
|
| 568 |
+
_shutdown = getattr(self.sock, "shutdown", None)
|
| 569 |
+
|
| 570 |
+
# Get the response from http.client.HTTPConnection
|
| 571 |
+
httplib_response = super().getresponse()
|
| 572 |
+
|
| 573 |
+
try:
|
| 574 |
+
assert_header_parsing(httplib_response.msg)
|
| 575 |
+
except (HeaderParsingError, TypeError) as hpe:
|
| 576 |
+
log.warning(
|
| 577 |
+
"Failed to parse headers (url=%s): %s",
|
| 578 |
+
_url_from_connection(self, resp_options.request_url),
|
| 579 |
+
hpe,
|
| 580 |
+
exc_info=True,
|
| 581 |
+
)
|
| 582 |
+
|
| 583 |
+
headers = HTTPHeaderDict(httplib_response.msg.items())
|
| 584 |
+
|
| 585 |
+
response = HTTPResponse(
|
| 586 |
+
body=httplib_response,
|
| 587 |
+
headers=headers,
|
| 588 |
+
status=httplib_response.status,
|
| 589 |
+
version=httplib_response.version,
|
| 590 |
+
version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
|
| 591 |
+
reason=httplib_response.reason,
|
| 592 |
+
preload_content=resp_options.preload_content,
|
| 593 |
+
decode_content=resp_options.decode_content,
|
| 594 |
+
original_response=httplib_response,
|
| 595 |
+
enforce_content_length=resp_options.enforce_content_length,
|
| 596 |
+
request_method=resp_options.request_method,
|
| 597 |
+
request_url=resp_options.request_url,
|
| 598 |
+
sock_shutdown=_shutdown,
|
| 599 |
+
)
|
| 600 |
+
return response
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
class HTTPSConnection(HTTPConnection):
|
| 604 |
+
"""
|
| 605 |
+
Many of the parameters to this constructor are passed to the underlying SSL
|
| 606 |
+
socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
|
| 607 |
+
"""
|
| 608 |
+
|
| 609 |
+
default_port = port_by_scheme["https"] # type: ignore[misc]
|
| 610 |
+
|
| 611 |
+
cert_reqs: int | str | None = None
|
| 612 |
+
ca_certs: str | None = None
|
| 613 |
+
ca_cert_dir: str | None = None
|
| 614 |
+
ca_cert_data: None | str | bytes = None
|
| 615 |
+
ssl_version: int | str | None = None
|
| 616 |
+
ssl_minimum_version: int | None = None
|
| 617 |
+
ssl_maximum_version: int | None = None
|
| 618 |
+
assert_fingerprint: str | None = None
|
| 619 |
+
_connect_callback: typing.Callable[..., None] | None = None
|
| 620 |
+
|
| 621 |
+
def __init__(
|
| 622 |
+
self,
|
| 623 |
+
host: str,
|
| 624 |
+
port: int | None = None,
|
| 625 |
+
*,
|
| 626 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 627 |
+
source_address: tuple[str, int] | None = None,
|
| 628 |
+
blocksize: int = 16384,
|
| 629 |
+
socket_options: None | (
|
| 630 |
+
connection._TYPE_SOCKET_OPTIONS
|
| 631 |
+
) = HTTPConnection.default_socket_options,
|
| 632 |
+
proxy: Url | None = None,
|
| 633 |
+
proxy_config: ProxyConfig | None = None,
|
| 634 |
+
cert_reqs: int | str | None = None,
|
| 635 |
+
assert_hostname: None | str | typing.Literal[False] = None,
|
| 636 |
+
assert_fingerprint: str | None = None,
|
| 637 |
+
server_hostname: str | None = None,
|
| 638 |
+
ssl_context: ssl.SSLContext | None = None,
|
| 639 |
+
ca_certs: str | None = None,
|
| 640 |
+
ca_cert_dir: str | None = None,
|
| 641 |
+
ca_cert_data: None | str | bytes = None,
|
| 642 |
+
ssl_minimum_version: int | None = None,
|
| 643 |
+
ssl_maximum_version: int | None = None,
|
| 644 |
+
ssl_version: int | str | None = None, # Deprecated
|
| 645 |
+
cert_file: str | None = None,
|
| 646 |
+
key_file: str | None = None,
|
| 647 |
+
key_password: str | None = None,
|
| 648 |
+
) -> None:
|
| 649 |
+
super().__init__(
|
| 650 |
+
host,
|
| 651 |
+
port=port,
|
| 652 |
+
timeout=timeout,
|
| 653 |
+
source_address=source_address,
|
| 654 |
+
blocksize=blocksize,
|
| 655 |
+
socket_options=socket_options,
|
| 656 |
+
proxy=proxy,
|
| 657 |
+
proxy_config=proxy_config,
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
self.key_file = key_file
|
| 661 |
+
self.cert_file = cert_file
|
| 662 |
+
self.key_password = key_password
|
| 663 |
+
self.ssl_context = ssl_context
|
| 664 |
+
self.server_hostname = server_hostname
|
| 665 |
+
self.assert_hostname = assert_hostname
|
| 666 |
+
self.assert_fingerprint = assert_fingerprint
|
| 667 |
+
self.ssl_version = ssl_version
|
| 668 |
+
self.ssl_minimum_version = ssl_minimum_version
|
| 669 |
+
self.ssl_maximum_version = ssl_maximum_version
|
| 670 |
+
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
| 671 |
+
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
| 672 |
+
self.ca_cert_data = ca_cert_data
|
| 673 |
+
|
| 674 |
+
# cert_reqs depends on ssl_context so calculate last.
|
| 675 |
+
if cert_reqs is None:
|
| 676 |
+
if self.ssl_context is not None:
|
| 677 |
+
cert_reqs = self.ssl_context.verify_mode
|
| 678 |
+
else:
|
| 679 |
+
cert_reqs = resolve_cert_reqs(None)
|
| 680 |
+
self.cert_reqs = cert_reqs
|
| 681 |
+
self._connect_callback = None
|
| 682 |
+
|
| 683 |
+
def set_cert(
|
| 684 |
+
self,
|
| 685 |
+
key_file: str | None = None,
|
| 686 |
+
cert_file: str | None = None,
|
| 687 |
+
cert_reqs: int | str | None = None,
|
| 688 |
+
key_password: str | None = None,
|
| 689 |
+
ca_certs: str | None = None,
|
| 690 |
+
assert_hostname: None | str | typing.Literal[False] = None,
|
| 691 |
+
assert_fingerprint: str | None = None,
|
| 692 |
+
ca_cert_dir: str | None = None,
|
| 693 |
+
ca_cert_data: None | str | bytes = None,
|
| 694 |
+
) -> None:
|
| 695 |
+
"""
|
| 696 |
+
This method should only be called once, before the connection is used.
|
| 697 |
+
"""
|
| 698 |
+
warnings.warn(
|
| 699 |
+
"HTTPSConnection.set_cert() is deprecated and will be removed "
|
| 700 |
+
"in urllib3 v2.1.0. Instead provide the parameters to the "
|
| 701 |
+
"HTTPSConnection constructor.",
|
| 702 |
+
category=DeprecationWarning,
|
| 703 |
+
stacklevel=2,
|
| 704 |
+
)
|
| 705 |
+
|
| 706 |
+
# If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
|
| 707 |
+
# have an SSLContext object in which case we'll use its verify_mode.
|
| 708 |
+
if cert_reqs is None:
|
| 709 |
+
if self.ssl_context is not None:
|
| 710 |
+
cert_reqs = self.ssl_context.verify_mode
|
| 711 |
+
else:
|
| 712 |
+
cert_reqs = resolve_cert_reqs(None)
|
| 713 |
+
|
| 714 |
+
self.key_file = key_file
|
| 715 |
+
self.cert_file = cert_file
|
| 716 |
+
self.cert_reqs = cert_reqs
|
| 717 |
+
self.key_password = key_password
|
| 718 |
+
self.assert_hostname = assert_hostname
|
| 719 |
+
self.assert_fingerprint = assert_fingerprint
|
| 720 |
+
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
| 721 |
+
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
| 722 |
+
self.ca_cert_data = ca_cert_data
|
| 723 |
+
|
| 724 |
+
def connect(self) -> None:
|
| 725 |
+
# Today we don't need to be doing this step before the /actual/ socket
|
| 726 |
+
# connection, however in the future we'll need to decide whether to
|
| 727 |
+
# create a new socket or re-use an existing "shared" socket as a part
|
| 728 |
+
# of the HTTP/2 handshake dance.
|
| 729 |
+
if self._tunnel_host is not None and self._tunnel_port is not None:
|
| 730 |
+
probe_http2_host = self._tunnel_host
|
| 731 |
+
probe_http2_port = self._tunnel_port
|
| 732 |
+
else:
|
| 733 |
+
probe_http2_host = self.host
|
| 734 |
+
probe_http2_port = self.port
|
| 735 |
+
|
| 736 |
+
# Check if the target origin supports HTTP/2.
|
| 737 |
+
# If the value comes back as 'None' it means that the current thread
|
| 738 |
+
# is probing for HTTP/2 support. Otherwise, we're waiting for another
|
| 739 |
+
# probe to complete, or we get a value right away.
|
| 740 |
+
target_supports_http2: bool | None
|
| 741 |
+
if "h2" in ssl_.ALPN_PROTOCOLS:
|
| 742 |
+
target_supports_http2 = http2_probe.acquire_and_get(
|
| 743 |
+
host=probe_http2_host, port=probe_http2_port
|
| 744 |
+
)
|
| 745 |
+
else:
|
| 746 |
+
# If HTTP/2 isn't going to be offered it doesn't matter if
|
| 747 |
+
# the target supports HTTP/2. Don't want to make a probe.
|
| 748 |
+
target_supports_http2 = False
|
| 749 |
+
|
| 750 |
+
if self._connect_callback is not None:
|
| 751 |
+
self._connect_callback(
|
| 752 |
+
"before connect",
|
| 753 |
+
thread_id=threading.get_ident(),
|
| 754 |
+
target_supports_http2=target_supports_http2,
|
| 755 |
+
)
|
| 756 |
+
|
| 757 |
+
try:
|
| 758 |
+
sock: socket.socket | ssl.SSLSocket
|
| 759 |
+
self.sock = sock = self._new_conn()
|
| 760 |
+
server_hostname: str = self.host
|
| 761 |
+
tls_in_tls = False
|
| 762 |
+
|
| 763 |
+
# Do we need to establish a tunnel?
|
| 764 |
+
if self.proxy_is_tunneling:
|
| 765 |
+
# We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
|
| 766 |
+
if self._tunnel_scheme == "https":
|
| 767 |
+
# _connect_tls_proxy will verify and assign proxy_is_verified
|
| 768 |
+
self.sock = sock = self._connect_tls_proxy(self.host, sock)
|
| 769 |
+
tls_in_tls = True
|
| 770 |
+
elif self._tunnel_scheme == "http":
|
| 771 |
+
self.proxy_is_verified = False
|
| 772 |
+
|
| 773 |
+
# If we're tunneling it means we're connected to our proxy.
|
| 774 |
+
self._has_connected_to_proxy = True
|
| 775 |
+
|
| 776 |
+
self._tunnel()
|
| 777 |
+
# Override the host with the one we're requesting data from.
|
| 778 |
+
server_hostname = typing.cast(str, self._tunnel_host)
|
| 779 |
+
|
| 780 |
+
if self.server_hostname is not None:
|
| 781 |
+
server_hostname = self.server_hostname
|
| 782 |
+
|
| 783 |
+
is_time_off = datetime.date.today() < RECENT_DATE
|
| 784 |
+
if is_time_off:
|
| 785 |
+
warnings.warn(
|
| 786 |
+
(
|
| 787 |
+
f"System time is way off (before {RECENT_DATE}). This will probably "
|
| 788 |
+
"lead to SSL verification errors"
|
| 789 |
+
),
|
| 790 |
+
SystemTimeWarning,
|
| 791 |
+
)
|
| 792 |
+
|
| 793 |
+
# Remove trailing '.' from fqdn hostnames to allow certificate validation
|
| 794 |
+
server_hostname_rm_dot = server_hostname.rstrip(".")
|
| 795 |
+
|
| 796 |
+
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
|
| 797 |
+
sock=sock,
|
| 798 |
+
cert_reqs=self.cert_reqs,
|
| 799 |
+
ssl_version=self.ssl_version,
|
| 800 |
+
ssl_minimum_version=self.ssl_minimum_version,
|
| 801 |
+
ssl_maximum_version=self.ssl_maximum_version,
|
| 802 |
+
ca_certs=self.ca_certs,
|
| 803 |
+
ca_cert_dir=self.ca_cert_dir,
|
| 804 |
+
ca_cert_data=self.ca_cert_data,
|
| 805 |
+
cert_file=self.cert_file,
|
| 806 |
+
key_file=self.key_file,
|
| 807 |
+
key_password=self.key_password,
|
| 808 |
+
server_hostname=server_hostname_rm_dot,
|
| 809 |
+
ssl_context=self.ssl_context,
|
| 810 |
+
tls_in_tls=tls_in_tls,
|
| 811 |
+
assert_hostname=self.assert_hostname,
|
| 812 |
+
assert_fingerprint=self.assert_fingerprint,
|
| 813 |
+
)
|
| 814 |
+
self.sock = sock_and_verified.socket
|
| 815 |
+
|
| 816 |
+
# If an error occurs during connection/handshake we may need to release
|
| 817 |
+
# our lock so another connection can probe the origin.
|
| 818 |
+
except BaseException:
|
| 819 |
+
if self._connect_callback is not None:
|
| 820 |
+
self._connect_callback(
|
| 821 |
+
"after connect failure",
|
| 822 |
+
thread_id=threading.get_ident(),
|
| 823 |
+
target_supports_http2=target_supports_http2,
|
| 824 |
+
)
|
| 825 |
+
|
| 826 |
+
if target_supports_http2 is None:
|
| 827 |
+
http2_probe.set_and_release(
|
| 828 |
+
host=probe_http2_host, port=probe_http2_port, supports_http2=None
|
| 829 |
+
)
|
| 830 |
+
raise
|
| 831 |
+
|
| 832 |
+
# If this connection doesn't know if the origin supports HTTP/2
|
| 833 |
+
# we report back to the HTTP/2 probe our result.
|
| 834 |
+
if target_supports_http2 is None:
|
| 835 |
+
supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2"
|
| 836 |
+
http2_probe.set_and_release(
|
| 837 |
+
host=probe_http2_host,
|
| 838 |
+
port=probe_http2_port,
|
| 839 |
+
supports_http2=supports_http2,
|
| 840 |
+
)
|
| 841 |
+
|
| 842 |
+
# Forwarding proxies can never have a verified target since
|
| 843 |
+
# the proxy is the one doing the verification. Should instead
|
| 844 |
+
# use a CONNECT tunnel in order to verify the target.
|
| 845 |
+
# See: https://github.com/urllib3/urllib3/issues/3267.
|
| 846 |
+
if self.proxy_is_forwarding:
|
| 847 |
+
self.is_verified = False
|
| 848 |
+
else:
|
| 849 |
+
self.is_verified = sock_and_verified.is_verified
|
| 850 |
+
|
| 851 |
+
# If there's a proxy to be connected to we are fully connected.
|
| 852 |
+
# This is set twice (once above and here) due to forwarding proxies
|
| 853 |
+
# not using tunnelling.
|
| 854 |
+
self._has_connected_to_proxy = bool(self.proxy)
|
| 855 |
+
|
| 856 |
+
# Set `self.proxy_is_verified` unless it's already set while
|
| 857 |
+
# establishing a tunnel.
|
| 858 |
+
if self._has_connected_to_proxy and self.proxy_is_verified is None:
|
| 859 |
+
self.proxy_is_verified = sock_and_verified.is_verified
|
| 860 |
+
|
| 861 |
+
def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
|
| 862 |
+
"""
|
| 863 |
+
Establish a TLS connection to the proxy using the provided SSL context.
|
| 864 |
+
"""
|
| 865 |
+
# `_connect_tls_proxy` is called when self._tunnel_host is truthy.
|
| 866 |
+
proxy_config = typing.cast(ProxyConfig, self.proxy_config)
|
| 867 |
+
ssl_context = proxy_config.ssl_context
|
| 868 |
+
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
|
| 869 |
+
sock,
|
| 870 |
+
cert_reqs=self.cert_reqs,
|
| 871 |
+
ssl_version=self.ssl_version,
|
| 872 |
+
ssl_minimum_version=self.ssl_minimum_version,
|
| 873 |
+
ssl_maximum_version=self.ssl_maximum_version,
|
| 874 |
+
ca_certs=self.ca_certs,
|
| 875 |
+
ca_cert_dir=self.ca_cert_dir,
|
| 876 |
+
ca_cert_data=self.ca_cert_data,
|
| 877 |
+
server_hostname=hostname,
|
| 878 |
+
ssl_context=ssl_context,
|
| 879 |
+
assert_hostname=proxy_config.assert_hostname,
|
| 880 |
+
assert_fingerprint=proxy_config.assert_fingerprint,
|
| 881 |
+
# Features that aren't implemented for proxies yet:
|
| 882 |
+
cert_file=None,
|
| 883 |
+
key_file=None,
|
| 884 |
+
key_password=None,
|
| 885 |
+
tls_in_tls=False,
|
| 886 |
+
)
|
| 887 |
+
self.proxy_is_verified = sock_and_verified.is_verified
|
| 888 |
+
return sock_and_verified.socket # type: ignore[return-value]
|
| 889 |
+
|
| 890 |
+
|
| 891 |
+
class _WrappedAndVerifiedSocket(typing.NamedTuple):
|
| 892 |
+
"""
|
| 893 |
+
Wrapped socket and whether the connection is
|
| 894 |
+
verified after the TLS handshake
|
| 895 |
+
"""
|
| 896 |
+
|
| 897 |
+
socket: ssl.SSLSocket | SSLTransport
|
| 898 |
+
is_verified: bool
|
| 899 |
+
|
| 900 |
+
|
| 901 |
+
def _ssl_wrap_socket_and_match_hostname(
|
| 902 |
+
sock: socket.socket,
|
| 903 |
+
*,
|
| 904 |
+
cert_reqs: None | str | int,
|
| 905 |
+
ssl_version: None | str | int,
|
| 906 |
+
ssl_minimum_version: int | None,
|
| 907 |
+
ssl_maximum_version: int | None,
|
| 908 |
+
cert_file: str | None,
|
| 909 |
+
key_file: str | None,
|
| 910 |
+
key_password: str | None,
|
| 911 |
+
ca_certs: str | None,
|
| 912 |
+
ca_cert_dir: str | None,
|
| 913 |
+
ca_cert_data: None | str | bytes,
|
| 914 |
+
assert_hostname: None | str | typing.Literal[False],
|
| 915 |
+
assert_fingerprint: str | None,
|
| 916 |
+
server_hostname: str | None,
|
| 917 |
+
ssl_context: ssl.SSLContext | None,
|
| 918 |
+
tls_in_tls: bool = False,
|
| 919 |
+
) -> _WrappedAndVerifiedSocket:
|
| 920 |
+
"""Logic for constructing an SSLContext from all TLS parameters, passing
|
| 921 |
+
that down into ssl_wrap_socket, and then doing certificate verification
|
| 922 |
+
either via hostname or fingerprint. This function exists to guarantee
|
| 923 |
+
that both proxies and targets have the same behavior when connecting via TLS.
|
| 924 |
+
"""
|
| 925 |
+
default_ssl_context = False
|
| 926 |
+
if ssl_context is None:
|
| 927 |
+
default_ssl_context = True
|
| 928 |
+
context = create_urllib3_context(
|
| 929 |
+
ssl_version=resolve_ssl_version(ssl_version),
|
| 930 |
+
ssl_minimum_version=ssl_minimum_version,
|
| 931 |
+
ssl_maximum_version=ssl_maximum_version,
|
| 932 |
+
cert_reqs=resolve_cert_reqs(cert_reqs),
|
| 933 |
+
)
|
| 934 |
+
else:
|
| 935 |
+
context = ssl_context
|
| 936 |
+
|
| 937 |
+
context.verify_mode = resolve_cert_reqs(cert_reqs)
|
| 938 |
+
|
| 939 |
+
# In some cases, we want to verify hostnames ourselves
|
| 940 |
+
if (
|
| 941 |
+
# `ssl` can't verify fingerprints or alternate hostnames
|
| 942 |
+
assert_fingerprint
|
| 943 |
+
or assert_hostname
|
| 944 |
+
# assert_hostname can be set to False to disable hostname checking
|
| 945 |
+
or assert_hostname is False
|
| 946 |
+
# We still support OpenSSL 1.0.2, which prevents us from verifying
|
| 947 |
+
# hostnames easily: https://github.com/pyca/pyopenssl/pull/933
|
| 948 |
+
or ssl_.IS_PYOPENSSL
|
| 949 |
+
or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
|
| 950 |
+
):
|
| 951 |
+
context.check_hostname = False
|
| 952 |
+
|
| 953 |
+
# Try to load OS default certs if none are given. We need to do the hasattr() check
|
| 954 |
+
# for custom pyOpenSSL SSLContext objects because they don't support
|
| 955 |
+
# load_default_certs().
|
| 956 |
+
if (
|
| 957 |
+
not ca_certs
|
| 958 |
+
and not ca_cert_dir
|
| 959 |
+
and not ca_cert_data
|
| 960 |
+
and default_ssl_context
|
| 961 |
+
and hasattr(context, "load_default_certs")
|
| 962 |
+
):
|
| 963 |
+
context.load_default_certs()
|
| 964 |
+
|
| 965 |
+
# Ensure that IPv6 addresses are in the proper format and don't have a
|
| 966 |
+
# scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
|
| 967 |
+
# and interprets them as DNS hostnames.
|
| 968 |
+
if server_hostname is not None:
|
| 969 |
+
normalized = server_hostname.strip("[]")
|
| 970 |
+
if "%" in normalized:
|
| 971 |
+
normalized = normalized[: normalized.rfind("%")]
|
| 972 |
+
if is_ipaddress(normalized):
|
| 973 |
+
server_hostname = normalized
|
| 974 |
+
|
| 975 |
+
ssl_sock = ssl_wrap_socket(
|
| 976 |
+
sock=sock,
|
| 977 |
+
keyfile=key_file,
|
| 978 |
+
certfile=cert_file,
|
| 979 |
+
key_password=key_password,
|
| 980 |
+
ca_certs=ca_certs,
|
| 981 |
+
ca_cert_dir=ca_cert_dir,
|
| 982 |
+
ca_cert_data=ca_cert_data,
|
| 983 |
+
server_hostname=server_hostname,
|
| 984 |
+
ssl_context=context,
|
| 985 |
+
tls_in_tls=tls_in_tls,
|
| 986 |
+
)
|
| 987 |
+
|
| 988 |
+
try:
|
| 989 |
+
if assert_fingerprint:
|
| 990 |
+
_assert_fingerprint(
|
| 991 |
+
ssl_sock.getpeercert(binary_form=True), assert_fingerprint
|
| 992 |
+
)
|
| 993 |
+
elif (
|
| 994 |
+
context.verify_mode != ssl.CERT_NONE
|
| 995 |
+
and not context.check_hostname
|
| 996 |
+
and assert_hostname is not False
|
| 997 |
+
):
|
| 998 |
+
cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
|
| 999 |
+
|
| 1000 |
+
# Need to signal to our match_hostname whether to use 'commonName' or not.
|
| 1001 |
+
# If we're using our own constructed SSLContext we explicitly set 'False'
|
| 1002 |
+
# because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
|
| 1003 |
+
if default_ssl_context:
|
| 1004 |
+
hostname_checks_common_name = False
|
| 1005 |
+
else:
|
| 1006 |
+
hostname_checks_common_name = (
|
| 1007 |
+
getattr(context, "hostname_checks_common_name", False) or False
|
| 1008 |
+
)
|
| 1009 |
+
|
| 1010 |
+
_match_hostname(
|
| 1011 |
+
cert,
|
| 1012 |
+
assert_hostname or server_hostname, # type: ignore[arg-type]
|
| 1013 |
+
hostname_checks_common_name,
|
| 1014 |
+
)
|
| 1015 |
+
|
| 1016 |
+
return _WrappedAndVerifiedSocket(
|
| 1017 |
+
socket=ssl_sock,
|
| 1018 |
+
is_verified=context.verify_mode == ssl.CERT_REQUIRED
|
| 1019 |
+
or bool(assert_fingerprint),
|
| 1020 |
+
)
|
| 1021 |
+
except BaseException:
|
| 1022 |
+
ssl_sock.close()
|
| 1023 |
+
raise
|
| 1024 |
+
|
| 1025 |
+
|
| 1026 |
+
def _match_hostname(
|
| 1027 |
+
cert: _TYPE_PEER_CERT_RET_DICT | None,
|
| 1028 |
+
asserted_hostname: str,
|
| 1029 |
+
hostname_checks_common_name: bool = False,
|
| 1030 |
+
) -> None:
|
| 1031 |
+
# Our upstream implementation of ssl.match_hostname()
|
| 1032 |
+
# only applies this normalization to IP addresses so it doesn't
|
| 1033 |
+
# match DNS SANs so we do the same thing!
|
| 1034 |
+
stripped_hostname = asserted_hostname.strip("[]")
|
| 1035 |
+
if is_ipaddress(stripped_hostname):
|
| 1036 |
+
asserted_hostname = stripped_hostname
|
| 1037 |
+
|
| 1038 |
+
try:
|
| 1039 |
+
match_hostname(cert, asserted_hostname, hostname_checks_common_name)
|
| 1040 |
+
except CertificateError as e:
|
| 1041 |
+
log.warning(
|
| 1042 |
+
"Certificate did not match expected hostname: %s. Certificate: %s",
|
| 1043 |
+
asserted_hostname,
|
| 1044 |
+
cert,
|
| 1045 |
+
)
|
| 1046 |
+
# Add cert to exception and reraise so client code can inspect
|
| 1047 |
+
# the cert when catching the exception, if they want to
|
| 1048 |
+
e._peer_cert = cert # type: ignore[attr-defined]
|
| 1049 |
+
raise
|
| 1050 |
+
|
| 1051 |
+
|
| 1052 |
+
def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
|
| 1053 |
+
# Look for the phrase 'wrong version number', if found
|
| 1054 |
+
# then we should warn the user that we're very sure that
|
| 1055 |
+
# this proxy is HTTP-only and they have a configuration issue.
|
| 1056 |
+
error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
|
| 1057 |
+
is_likely_http_proxy = (
|
| 1058 |
+
"wrong version number" in error_normalized
|
| 1059 |
+
or "unknown protocol" in error_normalized
|
| 1060 |
+
or "record layer failure" in error_normalized
|
| 1061 |
+
)
|
| 1062 |
+
http_proxy_warning = (
|
| 1063 |
+
". Your proxy appears to only use HTTP and not HTTPS, "
|
| 1064 |
+
"try changing your proxy URL to be HTTP. See: "
|
| 1065 |
+
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
|
| 1066 |
+
"#https-proxy-error-http-proxy"
|
| 1067 |
+
)
|
| 1068 |
+
new_err = ProxyError(
|
| 1069 |
+
f"Unable to connect to proxy"
|
| 1070 |
+
f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
|
| 1071 |
+
err,
|
| 1072 |
+
)
|
| 1073 |
+
new_err.__cause__ = err
|
| 1074 |
+
return new_err
|
| 1075 |
+
|
| 1076 |
+
|
| 1077 |
+
def _get_default_user_agent() -> str:
|
| 1078 |
+
return f"python-urllib3/{__version__}"
|
| 1079 |
+
|
| 1080 |
+
|
| 1081 |
+
class DummyConnection:
|
| 1082 |
+
"""Used to detect a failed ConnectionCls import."""
|
| 1083 |
+
|
| 1084 |
+
|
| 1085 |
+
if not ssl:
|
| 1086 |
+
HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
|
| 1087 |
+
|
| 1088 |
+
|
| 1089 |
+
VerifiedHTTPSConnection = HTTPSConnection
|
| 1090 |
+
|
| 1091 |
+
|
| 1092 |
+
def _url_from_connection(
|
| 1093 |
+
conn: HTTPConnection | HTTPSConnection, path: str | None = None
|
| 1094 |
+
) -> str:
|
| 1095 |
+
"""Returns the URL from a given connection. This is mainly used for testing and logging."""
|
| 1096 |
+
|
| 1097 |
+
scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
|
| 1098 |
+
|
| 1099 |
+
return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/connectionpool.py
ADDED
|
@@ -0,0 +1,1178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import errno
|
| 4 |
+
import logging
|
| 5 |
+
import queue
|
| 6 |
+
import sys
|
| 7 |
+
import typing
|
| 8 |
+
import warnings
|
| 9 |
+
import weakref
|
| 10 |
+
from socket import timeout as SocketTimeout
|
| 11 |
+
from types import TracebackType
|
| 12 |
+
|
| 13 |
+
from ._base_connection import _TYPE_BODY
|
| 14 |
+
from ._collections import HTTPHeaderDict
|
| 15 |
+
from ._request_methods import RequestMethods
|
| 16 |
+
from .connection import (
|
| 17 |
+
BaseSSLError,
|
| 18 |
+
BrokenPipeError,
|
| 19 |
+
DummyConnection,
|
| 20 |
+
HTTPConnection,
|
| 21 |
+
HTTPException,
|
| 22 |
+
HTTPSConnection,
|
| 23 |
+
ProxyConfig,
|
| 24 |
+
_wrap_proxy_error,
|
| 25 |
+
)
|
| 26 |
+
from .connection import port_by_scheme as port_by_scheme
|
| 27 |
+
from .exceptions import (
|
| 28 |
+
ClosedPoolError,
|
| 29 |
+
EmptyPoolError,
|
| 30 |
+
FullPoolError,
|
| 31 |
+
HostChangedError,
|
| 32 |
+
InsecureRequestWarning,
|
| 33 |
+
LocationValueError,
|
| 34 |
+
MaxRetryError,
|
| 35 |
+
NewConnectionError,
|
| 36 |
+
ProtocolError,
|
| 37 |
+
ProxyError,
|
| 38 |
+
ReadTimeoutError,
|
| 39 |
+
SSLError,
|
| 40 |
+
TimeoutError,
|
| 41 |
+
)
|
| 42 |
+
from .response import BaseHTTPResponse
|
| 43 |
+
from .util.connection import is_connection_dropped
|
| 44 |
+
from .util.proxy import connection_requires_http_tunnel
|
| 45 |
+
from .util.request import _TYPE_BODY_POSITION, set_file_position
|
| 46 |
+
from .util.retry import Retry
|
| 47 |
+
from .util.ssl_match_hostname import CertificateError
|
| 48 |
+
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
|
| 49 |
+
from .util.url import Url, _encode_target
|
| 50 |
+
from .util.url import _normalize_host as normalize_host
|
| 51 |
+
from .util.url import parse_url
|
| 52 |
+
from .util.util import to_str
|
| 53 |
+
|
| 54 |
+
if typing.TYPE_CHECKING:
|
| 55 |
+
import ssl
|
| 56 |
+
|
| 57 |
+
from typing_extensions import Self
|
| 58 |
+
|
| 59 |
+
from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
|
| 60 |
+
|
| 61 |
+
log = logging.getLogger(__name__)
|
| 62 |
+
|
| 63 |
+
_TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# Pool objects
|
| 67 |
+
class ConnectionPool:
|
| 68 |
+
"""
|
| 69 |
+
Base class for all connection pools, such as
|
| 70 |
+
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
|
| 71 |
+
|
| 72 |
+
.. note::
|
| 73 |
+
ConnectionPool.urlopen() does not normalize or percent-encode target URIs
|
| 74 |
+
which is useful if your target server doesn't support percent-encoded
|
| 75 |
+
target URIs.
|
| 76 |
+
"""
|
| 77 |
+
|
| 78 |
+
scheme: str | None = None
|
| 79 |
+
QueueCls = queue.LifoQueue
|
| 80 |
+
|
| 81 |
+
def __init__(self, host: str, port: int | None = None) -> None:
|
| 82 |
+
if not host:
|
| 83 |
+
raise LocationValueError("No host specified.")
|
| 84 |
+
|
| 85 |
+
self.host = _normalize_host(host, scheme=self.scheme)
|
| 86 |
+
self.port = port
|
| 87 |
+
|
| 88 |
+
# This property uses 'normalize_host()' (not '_normalize_host()')
|
| 89 |
+
# to avoid removing square braces around IPv6 addresses.
|
| 90 |
+
# This value is sent to `HTTPConnection.set_tunnel()` if called
|
| 91 |
+
# because square braces are required for HTTP CONNECT tunneling.
|
| 92 |
+
self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()
|
| 93 |
+
|
| 94 |
+
def __str__(self) -> str:
|
| 95 |
+
return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
|
| 96 |
+
|
| 97 |
+
def __enter__(self) -> Self:
|
| 98 |
+
return self
|
| 99 |
+
|
| 100 |
+
def __exit__(
|
| 101 |
+
self,
|
| 102 |
+
exc_type: type[BaseException] | None,
|
| 103 |
+
exc_val: BaseException | None,
|
| 104 |
+
exc_tb: TracebackType | None,
|
| 105 |
+
) -> typing.Literal[False]:
|
| 106 |
+
self.close()
|
| 107 |
+
# Return False to re-raise any potential exceptions
|
| 108 |
+
return False
|
| 109 |
+
|
| 110 |
+
def close(self) -> None:
|
| 111 |
+
"""
|
| 112 |
+
Close all pooled connections and disable the pool.
|
| 113 |
+
"""
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
|
| 117 |
+
_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
| 121 |
+
"""
|
| 122 |
+
Thread-safe connection pool for one host.
|
| 123 |
+
|
| 124 |
+
:param host:
|
| 125 |
+
Host used for this HTTP Connection (e.g. "localhost"), passed into
|
| 126 |
+
:class:`http.client.HTTPConnection`.
|
| 127 |
+
|
| 128 |
+
:param port:
|
| 129 |
+
Port used for this HTTP Connection (None is equivalent to 80), passed
|
| 130 |
+
into :class:`http.client.HTTPConnection`.
|
| 131 |
+
|
| 132 |
+
:param timeout:
|
| 133 |
+
Socket timeout in seconds for each individual connection. This can
|
| 134 |
+
be a float or integer, which sets the timeout for the HTTP request,
|
| 135 |
+
or an instance of :class:`urllib3.util.Timeout` which gives you more
|
| 136 |
+
fine-grained control over request timeouts. After the constructor has
|
| 137 |
+
been parsed, this is always a `urllib3.util.Timeout` object.
|
| 138 |
+
|
| 139 |
+
:param maxsize:
|
| 140 |
+
Number of connections to save that can be reused. More than 1 is useful
|
| 141 |
+
in multithreaded situations. If ``block`` is set to False, more
|
| 142 |
+
connections will be created but they will not be saved once they've
|
| 143 |
+
been used.
|
| 144 |
+
|
| 145 |
+
:param block:
|
| 146 |
+
If set to True, no more than ``maxsize`` connections will be used at
|
| 147 |
+
a time. When no free connections are available, the call will block
|
| 148 |
+
until a connection has been released. This is a useful side effect for
|
| 149 |
+
particular multithreaded situations where one does not want to use more
|
| 150 |
+
than maxsize connections per host to prevent flooding.
|
| 151 |
+
|
| 152 |
+
:param headers:
|
| 153 |
+
Headers to include with all requests, unless other headers are given
|
| 154 |
+
explicitly.
|
| 155 |
+
|
| 156 |
+
:param retries:
|
| 157 |
+
Retry configuration to use by default with requests in this pool.
|
| 158 |
+
|
| 159 |
+
:param _proxy:
|
| 160 |
+
Parsed proxy URL, should not be used directly, instead, see
|
| 161 |
+
:class:`urllib3.ProxyManager`
|
| 162 |
+
|
| 163 |
+
:param _proxy_headers:
|
| 164 |
+
A dictionary with proxy headers, should not be used directly,
|
| 165 |
+
instead, see :class:`urllib3.ProxyManager`
|
| 166 |
+
|
| 167 |
+
:param \\**conn_kw:
|
| 168 |
+
Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
|
| 169 |
+
:class:`urllib3.connection.HTTPSConnection` instances.
|
| 170 |
+
"""
|
| 171 |
+
|
| 172 |
+
scheme = "http"
|
| 173 |
+
ConnectionCls: type[BaseHTTPConnection] | type[BaseHTTPSConnection] = HTTPConnection
|
| 174 |
+
|
| 175 |
+
def __init__(
|
| 176 |
+
self,
|
| 177 |
+
host: str,
|
| 178 |
+
port: int | None = None,
|
| 179 |
+
timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
|
| 180 |
+
maxsize: int = 1,
|
| 181 |
+
block: bool = False,
|
| 182 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 183 |
+
retries: Retry | bool | int | None = None,
|
| 184 |
+
_proxy: Url | None = None,
|
| 185 |
+
_proxy_headers: typing.Mapping[str, str] | None = None,
|
| 186 |
+
_proxy_config: ProxyConfig | None = None,
|
| 187 |
+
**conn_kw: typing.Any,
|
| 188 |
+
):
|
| 189 |
+
ConnectionPool.__init__(self, host, port)
|
| 190 |
+
RequestMethods.__init__(self, headers)
|
| 191 |
+
|
| 192 |
+
if not isinstance(timeout, Timeout):
|
| 193 |
+
timeout = Timeout.from_float(timeout)
|
| 194 |
+
|
| 195 |
+
if retries is None:
|
| 196 |
+
retries = Retry.DEFAULT
|
| 197 |
+
|
| 198 |
+
self.timeout = timeout
|
| 199 |
+
self.retries = retries
|
| 200 |
+
|
| 201 |
+
self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)
|
| 202 |
+
self.block = block
|
| 203 |
+
|
| 204 |
+
self.proxy = _proxy
|
| 205 |
+
self.proxy_headers = _proxy_headers or {}
|
| 206 |
+
self.proxy_config = _proxy_config
|
| 207 |
+
|
| 208 |
+
# Fill the queue up so that doing get() on it will block properly
|
| 209 |
+
for _ in range(maxsize):
|
| 210 |
+
self.pool.put(None)
|
| 211 |
+
|
| 212 |
+
# These are mostly for testing and debugging purposes.
|
| 213 |
+
self.num_connections = 0
|
| 214 |
+
self.num_requests = 0
|
| 215 |
+
self.conn_kw = conn_kw
|
| 216 |
+
|
| 217 |
+
if self.proxy:
|
| 218 |
+
# Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
|
| 219 |
+
# We cannot know if the user has added default socket options, so we cannot replace the
|
| 220 |
+
# list.
|
| 221 |
+
self.conn_kw.setdefault("socket_options", [])
|
| 222 |
+
|
| 223 |
+
self.conn_kw["proxy"] = self.proxy
|
| 224 |
+
self.conn_kw["proxy_config"] = self.proxy_config
|
| 225 |
+
|
| 226 |
+
# Do not pass 'self' as callback to 'finalize'.
|
| 227 |
+
# Then the 'finalize' would keep an endless living (leak) to self.
|
| 228 |
+
# By just passing a reference to the pool allows the garbage collector
|
| 229 |
+
# to free self if nobody else has a reference to it.
|
| 230 |
+
pool = self.pool
|
| 231 |
+
|
| 232 |
+
# Close all the HTTPConnections in the pool before the
|
| 233 |
+
# HTTPConnectionPool object is garbage collected.
|
| 234 |
+
weakref.finalize(self, _close_pool_connections, pool)
|
| 235 |
+
|
| 236 |
+
def _new_conn(self) -> BaseHTTPConnection:
|
| 237 |
+
"""
|
| 238 |
+
Return a fresh :class:`HTTPConnection`.
|
| 239 |
+
"""
|
| 240 |
+
self.num_connections += 1
|
| 241 |
+
log.debug(
|
| 242 |
+
"Starting new HTTP connection (%d): %s:%s",
|
| 243 |
+
self.num_connections,
|
| 244 |
+
self.host,
|
| 245 |
+
self.port or "80",
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
conn = self.ConnectionCls(
|
| 249 |
+
host=self.host,
|
| 250 |
+
port=self.port,
|
| 251 |
+
timeout=self.timeout.connect_timeout,
|
| 252 |
+
**self.conn_kw,
|
| 253 |
+
)
|
| 254 |
+
return conn
|
| 255 |
+
|
| 256 |
+
def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
|
| 257 |
+
"""
|
| 258 |
+
Get a connection. Will return a pooled connection if one is available.
|
| 259 |
+
|
| 260 |
+
If no connections are available and :prop:`.block` is ``False``, then a
|
| 261 |
+
fresh connection is returned.
|
| 262 |
+
|
| 263 |
+
:param timeout:
|
| 264 |
+
Seconds to wait before giving up and raising
|
| 265 |
+
:class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
|
| 266 |
+
:prop:`.block` is ``True``.
|
| 267 |
+
"""
|
| 268 |
+
conn = None
|
| 269 |
+
|
| 270 |
+
if self.pool is None:
|
| 271 |
+
raise ClosedPoolError(self, "Pool is closed.")
|
| 272 |
+
|
| 273 |
+
try:
|
| 274 |
+
conn = self.pool.get(block=self.block, timeout=timeout)
|
| 275 |
+
|
| 276 |
+
except AttributeError: # self.pool is None
|
| 277 |
+
raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
|
| 278 |
+
|
| 279 |
+
except queue.Empty:
|
| 280 |
+
if self.block:
|
| 281 |
+
raise EmptyPoolError(
|
| 282 |
+
self,
|
| 283 |
+
"Pool is empty and a new connection can't be opened due to blocking mode.",
|
| 284 |
+
) from None
|
| 285 |
+
pass # Oh well, we'll create a new connection then
|
| 286 |
+
|
| 287 |
+
# If this is a persistent connection, check if it got disconnected
|
| 288 |
+
if conn and is_connection_dropped(conn):
|
| 289 |
+
log.debug("Resetting dropped connection: %s", self.host)
|
| 290 |
+
conn.close()
|
| 291 |
+
|
| 292 |
+
return conn or self._new_conn()
|
| 293 |
+
|
| 294 |
+
def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
|
| 295 |
+
"""
|
| 296 |
+
Put a connection back into the pool.
|
| 297 |
+
|
| 298 |
+
:param conn:
|
| 299 |
+
Connection object for the current host and port as returned by
|
| 300 |
+
:meth:`._new_conn` or :meth:`._get_conn`.
|
| 301 |
+
|
| 302 |
+
If the pool is already full, the connection is closed and discarded
|
| 303 |
+
because we exceeded maxsize. If connections are discarded frequently,
|
| 304 |
+
then maxsize should be increased.
|
| 305 |
+
|
| 306 |
+
If the pool is closed, then the connection will be closed and discarded.
|
| 307 |
+
"""
|
| 308 |
+
if self.pool is not None:
|
| 309 |
+
try:
|
| 310 |
+
self.pool.put(conn, block=False)
|
| 311 |
+
return # Everything is dandy, done.
|
| 312 |
+
except AttributeError:
|
| 313 |
+
# self.pool is None.
|
| 314 |
+
pass
|
| 315 |
+
except queue.Full:
|
| 316 |
+
# Connection never got put back into the pool, close it.
|
| 317 |
+
if conn:
|
| 318 |
+
conn.close()
|
| 319 |
+
|
| 320 |
+
if self.block:
|
| 321 |
+
# This should never happen if you got the conn from self._get_conn
|
| 322 |
+
raise FullPoolError(
|
| 323 |
+
self,
|
| 324 |
+
"Pool reached maximum size and no more connections are allowed.",
|
| 325 |
+
) from None
|
| 326 |
+
|
| 327 |
+
log.warning(
|
| 328 |
+
"Connection pool is full, discarding connection: %s. Connection pool size: %s",
|
| 329 |
+
self.host,
|
| 330 |
+
self.pool.qsize(),
|
| 331 |
+
)
|
| 332 |
+
|
| 333 |
+
# Connection never got put back into the pool, close it.
|
| 334 |
+
if conn:
|
| 335 |
+
conn.close()
|
| 336 |
+
|
| 337 |
+
def _validate_conn(self, conn: BaseHTTPConnection) -> None:
|
| 338 |
+
"""
|
| 339 |
+
Called right before a request is made, after the socket is created.
|
| 340 |
+
"""
|
| 341 |
+
|
| 342 |
+
def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
|
| 343 |
+
# Nothing to do for HTTP connections.
|
| 344 |
+
pass
|
| 345 |
+
|
| 346 |
+
def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
|
| 347 |
+
"""Helper that always returns a :class:`urllib3.util.Timeout`"""
|
| 348 |
+
if timeout is _DEFAULT_TIMEOUT:
|
| 349 |
+
return self.timeout.clone()
|
| 350 |
+
|
| 351 |
+
if isinstance(timeout, Timeout):
|
| 352 |
+
return timeout.clone()
|
| 353 |
+
else:
|
| 354 |
+
# User passed us an int/float. This is for backwards compatibility,
|
| 355 |
+
# can be removed later
|
| 356 |
+
return Timeout.from_float(timeout)
|
| 357 |
+
|
| 358 |
+
def _raise_timeout(
|
| 359 |
+
self,
|
| 360 |
+
err: BaseSSLError | OSError | SocketTimeout,
|
| 361 |
+
url: str,
|
| 362 |
+
timeout_value: _TYPE_TIMEOUT | None,
|
| 363 |
+
) -> None:
|
| 364 |
+
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
|
| 365 |
+
|
| 366 |
+
if isinstance(err, SocketTimeout):
|
| 367 |
+
raise ReadTimeoutError(
|
| 368 |
+
self, url, f"Read timed out. (read timeout={timeout_value})"
|
| 369 |
+
) from err
|
| 370 |
+
|
| 371 |
+
# See the above comment about EAGAIN in Python 3.
|
| 372 |
+
if hasattr(err, "errno") and err.errno in _blocking_errnos:
|
| 373 |
+
raise ReadTimeoutError(
|
| 374 |
+
self, url, f"Read timed out. (read timeout={timeout_value})"
|
| 375 |
+
) from err
|
| 376 |
+
|
| 377 |
+
def _make_request(
|
| 378 |
+
self,
|
| 379 |
+
conn: BaseHTTPConnection,
|
| 380 |
+
method: str,
|
| 381 |
+
url: str,
|
| 382 |
+
body: _TYPE_BODY | None = None,
|
| 383 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 384 |
+
retries: Retry | None = None,
|
| 385 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 386 |
+
chunked: bool = False,
|
| 387 |
+
response_conn: BaseHTTPConnection | None = None,
|
| 388 |
+
preload_content: bool = True,
|
| 389 |
+
decode_content: bool = True,
|
| 390 |
+
enforce_content_length: bool = True,
|
| 391 |
+
) -> BaseHTTPResponse:
|
| 392 |
+
"""
|
| 393 |
+
Perform a request on a given urllib connection object taken from our
|
| 394 |
+
pool.
|
| 395 |
+
|
| 396 |
+
:param conn:
|
| 397 |
+
a connection from one of our connection pools
|
| 398 |
+
|
| 399 |
+
:param method:
|
| 400 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 401 |
+
|
| 402 |
+
:param url:
|
| 403 |
+
The URL to perform the request on.
|
| 404 |
+
|
| 405 |
+
:param body:
|
| 406 |
+
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
| 407 |
+
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
| 408 |
+
|
| 409 |
+
:param headers:
|
| 410 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 411 |
+
If-None-Match, etc. If None, pool headers are used. If provided,
|
| 412 |
+
these headers completely replace any pool-specific headers.
|
| 413 |
+
|
| 414 |
+
:param retries:
|
| 415 |
+
Configure the number of retries to allow before raising a
|
| 416 |
+
:class:`~urllib3.exceptions.MaxRetryError` exception.
|
| 417 |
+
|
| 418 |
+
Pass ``None`` to retry until you receive a response. Pass a
|
| 419 |
+
:class:`~urllib3.util.retry.Retry` object for fine-grained control
|
| 420 |
+
over different types of retries.
|
| 421 |
+
Pass an integer number to retry connection errors that many times,
|
| 422 |
+
but no other types of errors. Pass zero to never retry.
|
| 423 |
+
|
| 424 |
+
If ``False``, then retries are disabled and any exception is raised
|
| 425 |
+
immediately. Also, instead of raising a MaxRetryError on redirects,
|
| 426 |
+
the redirect response will be returned.
|
| 427 |
+
|
| 428 |
+
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
|
| 429 |
+
|
| 430 |
+
:param timeout:
|
| 431 |
+
If specified, overrides the default timeout for this one
|
| 432 |
+
request. It may be a float (in seconds) or an instance of
|
| 433 |
+
:class:`urllib3.util.Timeout`.
|
| 434 |
+
|
| 435 |
+
:param chunked:
|
| 436 |
+
If True, urllib3 will send the body using chunked transfer
|
| 437 |
+
encoding. Otherwise, urllib3 will send the body using the standard
|
| 438 |
+
content-length form. Defaults to False.
|
| 439 |
+
|
| 440 |
+
:param response_conn:
|
| 441 |
+
Set this to ``None`` if you will handle releasing the connection or
|
| 442 |
+
set the connection to have the response release it.
|
| 443 |
+
|
| 444 |
+
:param preload_content:
|
| 445 |
+
If True, the response's body will be preloaded during construction.
|
| 446 |
+
|
| 447 |
+
:param decode_content:
|
| 448 |
+
If True, will attempt to decode the body based on the
|
| 449 |
+
'content-encoding' header.
|
| 450 |
+
|
| 451 |
+
:param enforce_content_length:
|
| 452 |
+
Enforce content length checking. Body returned by server must match
|
| 453 |
+
value of Content-Length header, if present. Otherwise, raise error.
|
| 454 |
+
"""
|
| 455 |
+
self.num_requests += 1
|
| 456 |
+
|
| 457 |
+
timeout_obj = self._get_timeout(timeout)
|
| 458 |
+
timeout_obj.start_connect()
|
| 459 |
+
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
|
| 460 |
+
|
| 461 |
+
try:
|
| 462 |
+
# Trigger any extra validation we need to do.
|
| 463 |
+
try:
|
| 464 |
+
self._validate_conn(conn)
|
| 465 |
+
except (SocketTimeout, BaseSSLError) as e:
|
| 466 |
+
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
|
| 467 |
+
raise
|
| 468 |
+
|
| 469 |
+
# _validate_conn() starts the connection to an HTTPS proxy
|
| 470 |
+
# so we need to wrap errors with 'ProxyError' here too.
|
| 471 |
+
except (
|
| 472 |
+
OSError,
|
| 473 |
+
NewConnectionError,
|
| 474 |
+
TimeoutError,
|
| 475 |
+
BaseSSLError,
|
| 476 |
+
CertificateError,
|
| 477 |
+
SSLError,
|
| 478 |
+
) as e:
|
| 479 |
+
new_e: Exception = e
|
| 480 |
+
if isinstance(e, (BaseSSLError, CertificateError)):
|
| 481 |
+
new_e = SSLError(e)
|
| 482 |
+
# If the connection didn't successfully connect to it's proxy
|
| 483 |
+
# then there
|
| 484 |
+
if isinstance(
|
| 485 |
+
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
|
| 486 |
+
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
|
| 487 |
+
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
|
| 488 |
+
raise new_e
|
| 489 |
+
|
| 490 |
+
# conn.request() calls http.client.*.request, not the method in
|
| 491 |
+
# urllib3.request. It also calls makefile (recv) on the socket.
|
| 492 |
+
try:
|
| 493 |
+
conn.request(
|
| 494 |
+
method,
|
| 495 |
+
url,
|
| 496 |
+
body=body,
|
| 497 |
+
headers=headers,
|
| 498 |
+
chunked=chunked,
|
| 499 |
+
preload_content=preload_content,
|
| 500 |
+
decode_content=decode_content,
|
| 501 |
+
enforce_content_length=enforce_content_length,
|
| 502 |
+
)
|
| 503 |
+
|
| 504 |
+
# We are swallowing BrokenPipeError (errno.EPIPE) since the server is
|
| 505 |
+
# legitimately able to close the connection after sending a valid response.
|
| 506 |
+
# With this behaviour, the received response is still readable.
|
| 507 |
+
except BrokenPipeError:
|
| 508 |
+
pass
|
| 509 |
+
except OSError as e:
|
| 510 |
+
# MacOS/Linux
|
| 511 |
+
# EPROTOTYPE and ECONNRESET are needed on macOS
|
| 512 |
+
# https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
|
| 513 |
+
# Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
|
| 514 |
+
if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
|
| 515 |
+
raise
|
| 516 |
+
|
| 517 |
+
# Reset the timeout for the recv() on the socket
|
| 518 |
+
read_timeout = timeout_obj.read_timeout
|
| 519 |
+
|
| 520 |
+
if not conn.is_closed:
|
| 521 |
+
# In Python 3 socket.py will catch EAGAIN and return None when you
|
| 522 |
+
# try and read into the file pointer created by http.client, which
|
| 523 |
+
# instead raises a BadStatusLine exception. Instead of catching
|
| 524 |
+
# the exception and assuming all BadStatusLine exceptions are read
|
| 525 |
+
# timeouts, check for a zero timeout before making the request.
|
| 526 |
+
if read_timeout == 0:
|
| 527 |
+
raise ReadTimeoutError(
|
| 528 |
+
self, url, f"Read timed out. (read timeout={read_timeout})"
|
| 529 |
+
)
|
| 530 |
+
conn.timeout = read_timeout
|
| 531 |
+
|
| 532 |
+
# Receive the response from the server
|
| 533 |
+
try:
|
| 534 |
+
response = conn.getresponse()
|
| 535 |
+
except (BaseSSLError, OSError) as e:
|
| 536 |
+
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
|
| 537 |
+
raise
|
| 538 |
+
|
| 539 |
+
# Set properties that are used by the pooling layer.
|
| 540 |
+
response.retries = retries
|
| 541 |
+
response._connection = response_conn # type: ignore[attr-defined]
|
| 542 |
+
response._pool = self # type: ignore[attr-defined]
|
| 543 |
+
|
| 544 |
+
log.debug(
|
| 545 |
+
'%s://%s:%s "%s %s %s" %s %s',
|
| 546 |
+
self.scheme,
|
| 547 |
+
self.host,
|
| 548 |
+
self.port,
|
| 549 |
+
method,
|
| 550 |
+
url,
|
| 551 |
+
response.version_string,
|
| 552 |
+
response.status,
|
| 553 |
+
response.length_remaining,
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
return response
|
| 557 |
+
|
| 558 |
+
def close(self) -> None:
|
| 559 |
+
"""
|
| 560 |
+
Close all pooled connections and disable the pool.
|
| 561 |
+
"""
|
| 562 |
+
if self.pool is None:
|
| 563 |
+
return
|
| 564 |
+
# Disable access to the pool
|
| 565 |
+
old_pool, self.pool = self.pool, None
|
| 566 |
+
|
| 567 |
+
# Close all the HTTPConnections in the pool.
|
| 568 |
+
_close_pool_connections(old_pool)
|
| 569 |
+
|
| 570 |
+
def is_same_host(self, url: str) -> bool:
|
| 571 |
+
"""
|
| 572 |
+
Check if the given ``url`` is a member of the same host as this
|
| 573 |
+
connection pool.
|
| 574 |
+
"""
|
| 575 |
+
if url.startswith("/"):
|
| 576 |
+
return True
|
| 577 |
+
|
| 578 |
+
# TODO: Add optional support for socket.gethostbyname checking.
|
| 579 |
+
scheme, _, host, port, *_ = parse_url(url)
|
| 580 |
+
scheme = scheme or "http"
|
| 581 |
+
if host is not None:
|
| 582 |
+
host = _normalize_host(host, scheme=scheme)
|
| 583 |
+
|
| 584 |
+
# Use explicit default port for comparison when none is given
|
| 585 |
+
if self.port and not port:
|
| 586 |
+
port = port_by_scheme.get(scheme)
|
| 587 |
+
elif not self.port and port == port_by_scheme.get(scheme):
|
| 588 |
+
port = None
|
| 589 |
+
|
| 590 |
+
return (scheme, host, port) == (self.scheme, self.host, self.port)
|
| 591 |
+
|
| 592 |
+
def urlopen( # type: ignore[override]
|
| 593 |
+
self,
|
| 594 |
+
method: str,
|
| 595 |
+
url: str,
|
| 596 |
+
body: _TYPE_BODY | None = None,
|
| 597 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 598 |
+
retries: Retry | bool | int | None = None,
|
| 599 |
+
redirect: bool = True,
|
| 600 |
+
assert_same_host: bool = True,
|
| 601 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 602 |
+
pool_timeout: int | None = None,
|
| 603 |
+
release_conn: bool | None = None,
|
| 604 |
+
chunked: bool = False,
|
| 605 |
+
body_pos: _TYPE_BODY_POSITION | None = None,
|
| 606 |
+
preload_content: bool = True,
|
| 607 |
+
decode_content: bool = True,
|
| 608 |
+
**response_kw: typing.Any,
|
| 609 |
+
) -> BaseHTTPResponse:
|
| 610 |
+
"""
|
| 611 |
+
Get a connection from the pool and perform an HTTP request. This is the
|
| 612 |
+
lowest level call for making a request, so you'll need to specify all
|
| 613 |
+
the raw details.
|
| 614 |
+
|
| 615 |
+
.. note::
|
| 616 |
+
|
| 617 |
+
More commonly, it's appropriate to use a convenience method
|
| 618 |
+
such as :meth:`request`.
|
| 619 |
+
|
| 620 |
+
.. note::
|
| 621 |
+
|
| 622 |
+
`release_conn` will only behave as expected if
|
| 623 |
+
`preload_content=False` because we want to make
|
| 624 |
+
`preload_content=False` the default behaviour someday soon without
|
| 625 |
+
breaking backwards compatibility.
|
| 626 |
+
|
| 627 |
+
:param method:
|
| 628 |
+
HTTP request method (such as GET, POST, PUT, etc.)
|
| 629 |
+
|
| 630 |
+
:param url:
|
| 631 |
+
The URL to perform the request on.
|
| 632 |
+
|
| 633 |
+
:param body:
|
| 634 |
+
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
| 635 |
+
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
| 636 |
+
|
| 637 |
+
:param headers:
|
| 638 |
+
Dictionary of custom headers to send, such as User-Agent,
|
| 639 |
+
If-None-Match, etc. If None, pool headers are used. If provided,
|
| 640 |
+
these headers completely replace any pool-specific headers.
|
| 641 |
+
|
| 642 |
+
:param retries:
|
| 643 |
+
Configure the number of retries to allow before raising a
|
| 644 |
+
:class:`~urllib3.exceptions.MaxRetryError` exception.
|
| 645 |
+
|
| 646 |
+
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
|
| 647 |
+
:class:`~urllib3.util.retry.Retry` object for fine-grained control
|
| 648 |
+
over different types of retries.
|
| 649 |
+
Pass an integer number to retry connection errors that many times,
|
| 650 |
+
but no other types of errors. Pass zero to never retry.
|
| 651 |
+
|
| 652 |
+
If ``False``, then retries are disabled and any exception is raised
|
| 653 |
+
immediately. Also, instead of raising a MaxRetryError on redirects,
|
| 654 |
+
the redirect response will be returned.
|
| 655 |
+
|
| 656 |
+
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
|
| 657 |
+
|
| 658 |
+
:param redirect:
|
| 659 |
+
If True, automatically handle redirects (status codes 301, 302,
|
| 660 |
+
303, 307, 308). Each redirect counts as a retry. Disabling retries
|
| 661 |
+
will disable redirect, too.
|
| 662 |
+
|
| 663 |
+
:param assert_same_host:
|
| 664 |
+
If ``True``, will make sure that the host of the pool requests is
|
| 665 |
+
consistent else will raise HostChangedError. When ``False``, you can
|
| 666 |
+
use the pool on an HTTP proxy and request foreign hosts.
|
| 667 |
+
|
| 668 |
+
:param timeout:
|
| 669 |
+
If specified, overrides the default timeout for this one
|
| 670 |
+
request. It may be a float (in seconds) or an instance of
|
| 671 |
+
:class:`urllib3.util.Timeout`.
|
| 672 |
+
|
| 673 |
+
:param pool_timeout:
|
| 674 |
+
If set and the pool is set to block=True, then this method will
|
| 675 |
+
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
|
| 676 |
+
connection is available within the time period.
|
| 677 |
+
|
| 678 |
+
:param bool preload_content:
|
| 679 |
+
If True, the response's body will be preloaded into memory.
|
| 680 |
+
|
| 681 |
+
:param bool decode_content:
|
| 682 |
+
If True, will attempt to decode the body based on the
|
| 683 |
+
'content-encoding' header.
|
| 684 |
+
|
| 685 |
+
:param release_conn:
|
| 686 |
+
If False, then the urlopen call will not release the connection
|
| 687 |
+
back into the pool once a response is received (but will release if
|
| 688 |
+
you read the entire contents of the response such as when
|
| 689 |
+
`preload_content=True`). This is useful if you're not preloading
|
| 690 |
+
the response's content immediately. You will need to call
|
| 691 |
+
``r.release_conn()`` on the response ``r`` to return the connection
|
| 692 |
+
back into the pool. If None, it takes the value of ``preload_content``
|
| 693 |
+
which defaults to ``True``.
|
| 694 |
+
|
| 695 |
+
:param bool chunked:
|
| 696 |
+
If True, urllib3 will send the body using chunked transfer
|
| 697 |
+
encoding. Otherwise, urllib3 will send the body using the standard
|
| 698 |
+
content-length form. Defaults to False.
|
| 699 |
+
|
| 700 |
+
:param int body_pos:
|
| 701 |
+
Position to seek to in file-like body in the event of a retry or
|
| 702 |
+
redirect. Typically this won't need to be set because urllib3 will
|
| 703 |
+
auto-populate the value when needed.
|
| 704 |
+
"""
|
| 705 |
+
parsed_url = parse_url(url)
|
| 706 |
+
destination_scheme = parsed_url.scheme
|
| 707 |
+
|
| 708 |
+
if headers is None:
|
| 709 |
+
headers = self.headers
|
| 710 |
+
|
| 711 |
+
if not isinstance(retries, Retry):
|
| 712 |
+
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
|
| 713 |
+
|
| 714 |
+
if release_conn is None:
|
| 715 |
+
release_conn = preload_content
|
| 716 |
+
|
| 717 |
+
# Check host
|
| 718 |
+
if assert_same_host and not self.is_same_host(url):
|
| 719 |
+
raise HostChangedError(self, url, retries)
|
| 720 |
+
|
| 721 |
+
# Ensure that the URL we're connecting to is properly encoded
|
| 722 |
+
if url.startswith("/"):
|
| 723 |
+
url = to_str(_encode_target(url))
|
| 724 |
+
else:
|
| 725 |
+
url = to_str(parsed_url.url)
|
| 726 |
+
|
| 727 |
+
conn = None
|
| 728 |
+
|
| 729 |
+
# Track whether `conn` needs to be released before
|
| 730 |
+
# returning/raising/recursing. Update this variable if necessary, and
|
| 731 |
+
# leave `release_conn` constant throughout the function. That way, if
|
| 732 |
+
# the function recurses, the original value of `release_conn` will be
|
| 733 |
+
# passed down into the recursive call, and its value will be respected.
|
| 734 |
+
#
|
| 735 |
+
# See issue #651 [1] for details.
|
| 736 |
+
#
|
| 737 |
+
# [1] <https://github.com/urllib3/urllib3/issues/651>
|
| 738 |
+
release_this_conn = release_conn
|
| 739 |
+
|
| 740 |
+
http_tunnel_required = connection_requires_http_tunnel(
|
| 741 |
+
self.proxy, self.proxy_config, destination_scheme
|
| 742 |
+
)
|
| 743 |
+
|
| 744 |
+
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
|
| 745 |
+
# have to copy the headers dict so we can safely change it without those
|
| 746 |
+
# changes being reflected in anyone else's copy.
|
| 747 |
+
if not http_tunnel_required:
|
| 748 |
+
headers = headers.copy() # type: ignore[attr-defined]
|
| 749 |
+
headers.update(self.proxy_headers) # type: ignore[union-attr]
|
| 750 |
+
|
| 751 |
+
# Must keep the exception bound to a separate variable or else Python 3
|
| 752 |
+
# complains about UnboundLocalError.
|
| 753 |
+
err = None
|
| 754 |
+
|
| 755 |
+
# Keep track of whether we cleanly exited the except block. This
|
| 756 |
+
# ensures we do proper cleanup in finally.
|
| 757 |
+
clean_exit = False
|
| 758 |
+
|
| 759 |
+
# Rewind body position, if needed. Record current position
|
| 760 |
+
# for future rewinds in the event of a redirect/retry.
|
| 761 |
+
body_pos = set_file_position(body, body_pos)
|
| 762 |
+
|
| 763 |
+
try:
|
| 764 |
+
# Request a connection from the queue.
|
| 765 |
+
timeout_obj = self._get_timeout(timeout)
|
| 766 |
+
conn = self._get_conn(timeout=pool_timeout)
|
| 767 |
+
|
| 768 |
+
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
|
| 769 |
+
|
| 770 |
+
# Is this a closed/new connection that requires CONNECT tunnelling?
|
| 771 |
+
if self.proxy is not None and http_tunnel_required and conn.is_closed:
|
| 772 |
+
try:
|
| 773 |
+
self._prepare_proxy(conn)
|
| 774 |
+
except (BaseSSLError, OSError, SocketTimeout) as e:
|
| 775 |
+
self._raise_timeout(
|
| 776 |
+
err=e, url=self.proxy.url, timeout_value=conn.timeout
|
| 777 |
+
)
|
| 778 |
+
raise
|
| 779 |
+
|
| 780 |
+
# If we're going to release the connection in ``finally:``, then
|
| 781 |
+
# the response doesn't need to know about the connection. Otherwise
|
| 782 |
+
# it will also try to release it and we'll have a double-release
|
| 783 |
+
# mess.
|
| 784 |
+
response_conn = conn if not release_conn else None
|
| 785 |
+
|
| 786 |
+
# Make the request on the HTTPConnection object
|
| 787 |
+
response = self._make_request(
|
| 788 |
+
conn,
|
| 789 |
+
method,
|
| 790 |
+
url,
|
| 791 |
+
timeout=timeout_obj,
|
| 792 |
+
body=body,
|
| 793 |
+
headers=headers,
|
| 794 |
+
chunked=chunked,
|
| 795 |
+
retries=retries,
|
| 796 |
+
response_conn=response_conn,
|
| 797 |
+
preload_content=preload_content,
|
| 798 |
+
decode_content=decode_content,
|
| 799 |
+
**response_kw,
|
| 800 |
+
)
|
| 801 |
+
|
| 802 |
+
# Everything went great!
|
| 803 |
+
clean_exit = True
|
| 804 |
+
|
| 805 |
+
except EmptyPoolError:
|
| 806 |
+
# Didn't get a connection from the pool, no need to clean up
|
| 807 |
+
clean_exit = True
|
| 808 |
+
release_this_conn = False
|
| 809 |
+
raise
|
| 810 |
+
|
| 811 |
+
except (
|
| 812 |
+
TimeoutError,
|
| 813 |
+
HTTPException,
|
| 814 |
+
OSError,
|
| 815 |
+
ProtocolError,
|
| 816 |
+
BaseSSLError,
|
| 817 |
+
SSLError,
|
| 818 |
+
CertificateError,
|
| 819 |
+
ProxyError,
|
| 820 |
+
) as e:
|
| 821 |
+
# Discard the connection for these exceptions. It will be
|
| 822 |
+
# replaced during the next _get_conn() call.
|
| 823 |
+
clean_exit = False
|
| 824 |
+
new_e: Exception = e
|
| 825 |
+
if isinstance(e, (BaseSSLError, CertificateError)):
|
| 826 |
+
new_e = SSLError(e)
|
| 827 |
+
if isinstance(
|
| 828 |
+
new_e,
|
| 829 |
+
(
|
| 830 |
+
OSError,
|
| 831 |
+
NewConnectionError,
|
| 832 |
+
TimeoutError,
|
| 833 |
+
SSLError,
|
| 834 |
+
HTTPException,
|
| 835 |
+
),
|
| 836 |
+
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
|
| 837 |
+
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
|
| 838 |
+
elif isinstance(new_e, (OSError, HTTPException)):
|
| 839 |
+
new_e = ProtocolError("Connection aborted.", new_e)
|
| 840 |
+
|
| 841 |
+
retries = retries.increment(
|
| 842 |
+
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
|
| 843 |
+
)
|
| 844 |
+
retries.sleep()
|
| 845 |
+
|
| 846 |
+
# Keep track of the error for the retry warning.
|
| 847 |
+
err = e
|
| 848 |
+
|
| 849 |
+
finally:
|
| 850 |
+
if not clean_exit:
|
| 851 |
+
# We hit some kind of exception, handled or otherwise. We need
|
| 852 |
+
# to throw the connection away unless explicitly told not to.
|
| 853 |
+
# Close the connection, set the variable to None, and make sure
|
| 854 |
+
# we put the None back in the pool to avoid leaking it.
|
| 855 |
+
if conn:
|
| 856 |
+
conn.close()
|
| 857 |
+
conn = None
|
| 858 |
+
release_this_conn = True
|
| 859 |
+
|
| 860 |
+
if release_this_conn:
|
| 861 |
+
# Put the connection back to be reused. If the connection is
|
| 862 |
+
# expired then it will be None, which will get replaced with a
|
| 863 |
+
# fresh connection during _get_conn.
|
| 864 |
+
self._put_conn(conn)
|
| 865 |
+
|
| 866 |
+
if not conn:
|
| 867 |
+
# Try again
|
| 868 |
+
log.warning(
|
| 869 |
+
"Retrying (%r) after connection broken by '%r': %s", retries, err, url
|
| 870 |
+
)
|
| 871 |
+
return self.urlopen(
|
| 872 |
+
method,
|
| 873 |
+
url,
|
| 874 |
+
body,
|
| 875 |
+
headers,
|
| 876 |
+
retries,
|
| 877 |
+
redirect,
|
| 878 |
+
assert_same_host,
|
| 879 |
+
timeout=timeout,
|
| 880 |
+
pool_timeout=pool_timeout,
|
| 881 |
+
release_conn=release_conn,
|
| 882 |
+
chunked=chunked,
|
| 883 |
+
body_pos=body_pos,
|
| 884 |
+
preload_content=preload_content,
|
| 885 |
+
decode_content=decode_content,
|
| 886 |
+
**response_kw,
|
| 887 |
+
)
|
| 888 |
+
|
| 889 |
+
# Handle redirect?
|
| 890 |
+
redirect_location = redirect and response.get_redirect_location()
|
| 891 |
+
if redirect_location:
|
| 892 |
+
if response.status == 303:
|
| 893 |
+
# Change the method according to RFC 9110, Section 15.4.4.
|
| 894 |
+
method = "GET"
|
| 895 |
+
# And lose the body not to transfer anything sensitive.
|
| 896 |
+
body = None
|
| 897 |
+
headers = HTTPHeaderDict(headers)._prepare_for_method_change()
|
| 898 |
+
|
| 899 |
+
try:
|
| 900 |
+
retries = retries.increment(method, url, response=response, _pool=self)
|
| 901 |
+
except MaxRetryError:
|
| 902 |
+
if retries.raise_on_redirect:
|
| 903 |
+
response.drain_conn()
|
| 904 |
+
raise
|
| 905 |
+
return response
|
| 906 |
+
|
| 907 |
+
response.drain_conn()
|
| 908 |
+
retries.sleep_for_retry(response)
|
| 909 |
+
log.debug("Redirecting %s -> %s", url, redirect_location)
|
| 910 |
+
return self.urlopen(
|
| 911 |
+
method,
|
| 912 |
+
redirect_location,
|
| 913 |
+
body,
|
| 914 |
+
headers,
|
| 915 |
+
retries=retries,
|
| 916 |
+
redirect=redirect,
|
| 917 |
+
assert_same_host=assert_same_host,
|
| 918 |
+
timeout=timeout,
|
| 919 |
+
pool_timeout=pool_timeout,
|
| 920 |
+
release_conn=release_conn,
|
| 921 |
+
chunked=chunked,
|
| 922 |
+
body_pos=body_pos,
|
| 923 |
+
preload_content=preload_content,
|
| 924 |
+
decode_content=decode_content,
|
| 925 |
+
**response_kw,
|
| 926 |
+
)
|
| 927 |
+
|
| 928 |
+
# Check if we should retry the HTTP response.
|
| 929 |
+
has_retry_after = bool(response.headers.get("Retry-After"))
|
| 930 |
+
if retries.is_retry(method, response.status, has_retry_after):
|
| 931 |
+
try:
|
| 932 |
+
retries = retries.increment(method, url, response=response, _pool=self)
|
| 933 |
+
except MaxRetryError:
|
| 934 |
+
if retries.raise_on_status:
|
| 935 |
+
response.drain_conn()
|
| 936 |
+
raise
|
| 937 |
+
return response
|
| 938 |
+
|
| 939 |
+
response.drain_conn()
|
| 940 |
+
retries.sleep(response)
|
| 941 |
+
log.debug("Retry: %s", url)
|
| 942 |
+
return self.urlopen(
|
| 943 |
+
method,
|
| 944 |
+
url,
|
| 945 |
+
body,
|
| 946 |
+
headers,
|
| 947 |
+
retries=retries,
|
| 948 |
+
redirect=redirect,
|
| 949 |
+
assert_same_host=assert_same_host,
|
| 950 |
+
timeout=timeout,
|
| 951 |
+
pool_timeout=pool_timeout,
|
| 952 |
+
release_conn=release_conn,
|
| 953 |
+
chunked=chunked,
|
| 954 |
+
body_pos=body_pos,
|
| 955 |
+
preload_content=preload_content,
|
| 956 |
+
decode_content=decode_content,
|
| 957 |
+
**response_kw,
|
| 958 |
+
)
|
| 959 |
+
|
| 960 |
+
return response
|
| 961 |
+
|
| 962 |
+
|
| 963 |
+
class HTTPSConnectionPool(HTTPConnectionPool):
|
| 964 |
+
"""
|
| 965 |
+
Same as :class:`.HTTPConnectionPool`, but HTTPS.
|
| 966 |
+
|
| 967 |
+
:class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
|
| 968 |
+
``assert_hostname`` and ``host`` in this order to verify connections.
|
| 969 |
+
If ``assert_hostname`` is False, no verification is done.
|
| 970 |
+
|
| 971 |
+
The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
|
| 972 |
+
``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
|
| 973 |
+
is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
|
| 974 |
+
the connection socket into an SSL socket.
|
| 975 |
+
"""
|
| 976 |
+
|
| 977 |
+
scheme = "https"
|
| 978 |
+
ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
|
| 979 |
+
|
| 980 |
+
def __init__(
|
| 981 |
+
self,
|
| 982 |
+
host: str,
|
| 983 |
+
port: int | None = None,
|
| 984 |
+
timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
|
| 985 |
+
maxsize: int = 1,
|
| 986 |
+
block: bool = False,
|
| 987 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 988 |
+
retries: Retry | bool | int | None = None,
|
| 989 |
+
_proxy: Url | None = None,
|
| 990 |
+
_proxy_headers: typing.Mapping[str, str] | None = None,
|
| 991 |
+
key_file: str | None = None,
|
| 992 |
+
cert_file: str | None = None,
|
| 993 |
+
cert_reqs: int | str | None = None,
|
| 994 |
+
key_password: str | None = None,
|
| 995 |
+
ca_certs: str | None = None,
|
| 996 |
+
ssl_version: int | str | None = None,
|
| 997 |
+
ssl_minimum_version: ssl.TLSVersion | None = None,
|
| 998 |
+
ssl_maximum_version: ssl.TLSVersion | None = None,
|
| 999 |
+
assert_hostname: str | typing.Literal[False] | None = None,
|
| 1000 |
+
assert_fingerprint: str | None = None,
|
| 1001 |
+
ca_cert_dir: str | None = None,
|
| 1002 |
+
**conn_kw: typing.Any,
|
| 1003 |
+
) -> None:
|
| 1004 |
+
super().__init__(
|
| 1005 |
+
host,
|
| 1006 |
+
port,
|
| 1007 |
+
timeout,
|
| 1008 |
+
maxsize,
|
| 1009 |
+
block,
|
| 1010 |
+
headers,
|
| 1011 |
+
retries,
|
| 1012 |
+
_proxy,
|
| 1013 |
+
_proxy_headers,
|
| 1014 |
+
**conn_kw,
|
| 1015 |
+
)
|
| 1016 |
+
|
| 1017 |
+
self.key_file = key_file
|
| 1018 |
+
self.cert_file = cert_file
|
| 1019 |
+
self.cert_reqs = cert_reqs
|
| 1020 |
+
self.key_password = key_password
|
| 1021 |
+
self.ca_certs = ca_certs
|
| 1022 |
+
self.ca_cert_dir = ca_cert_dir
|
| 1023 |
+
self.ssl_version = ssl_version
|
| 1024 |
+
self.ssl_minimum_version = ssl_minimum_version
|
| 1025 |
+
self.ssl_maximum_version = ssl_maximum_version
|
| 1026 |
+
self.assert_hostname = assert_hostname
|
| 1027 |
+
self.assert_fingerprint = assert_fingerprint
|
| 1028 |
+
|
| 1029 |
+
def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
|
| 1030 |
+
"""Establishes a tunnel connection through HTTP CONNECT."""
|
| 1031 |
+
if self.proxy and self.proxy.scheme == "https":
|
| 1032 |
+
tunnel_scheme = "https"
|
| 1033 |
+
else:
|
| 1034 |
+
tunnel_scheme = "http"
|
| 1035 |
+
|
| 1036 |
+
conn.set_tunnel(
|
| 1037 |
+
scheme=tunnel_scheme,
|
| 1038 |
+
host=self._tunnel_host,
|
| 1039 |
+
port=self.port,
|
| 1040 |
+
headers=self.proxy_headers,
|
| 1041 |
+
)
|
| 1042 |
+
conn.connect()
|
| 1043 |
+
|
| 1044 |
+
def _new_conn(self) -> BaseHTTPSConnection:
|
| 1045 |
+
"""
|
| 1046 |
+
Return a fresh :class:`urllib3.connection.HTTPConnection`.
|
| 1047 |
+
"""
|
| 1048 |
+
self.num_connections += 1
|
| 1049 |
+
log.debug(
|
| 1050 |
+
"Starting new HTTPS connection (%d): %s:%s",
|
| 1051 |
+
self.num_connections,
|
| 1052 |
+
self.host,
|
| 1053 |
+
self.port or "443",
|
| 1054 |
+
)
|
| 1055 |
+
|
| 1056 |
+
if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
|
| 1057 |
+
raise ImportError(
|
| 1058 |
+
"Can't connect to HTTPS URL because the SSL module is not available."
|
| 1059 |
+
)
|
| 1060 |
+
|
| 1061 |
+
actual_host: str = self.host
|
| 1062 |
+
actual_port = self.port
|
| 1063 |
+
if self.proxy is not None and self.proxy.host is not None:
|
| 1064 |
+
actual_host = self.proxy.host
|
| 1065 |
+
actual_port = self.proxy.port
|
| 1066 |
+
|
| 1067 |
+
return self.ConnectionCls(
|
| 1068 |
+
host=actual_host,
|
| 1069 |
+
port=actual_port,
|
| 1070 |
+
timeout=self.timeout.connect_timeout,
|
| 1071 |
+
cert_file=self.cert_file,
|
| 1072 |
+
key_file=self.key_file,
|
| 1073 |
+
key_password=self.key_password,
|
| 1074 |
+
cert_reqs=self.cert_reqs,
|
| 1075 |
+
ca_certs=self.ca_certs,
|
| 1076 |
+
ca_cert_dir=self.ca_cert_dir,
|
| 1077 |
+
assert_hostname=self.assert_hostname,
|
| 1078 |
+
assert_fingerprint=self.assert_fingerprint,
|
| 1079 |
+
ssl_version=self.ssl_version,
|
| 1080 |
+
ssl_minimum_version=self.ssl_minimum_version,
|
| 1081 |
+
ssl_maximum_version=self.ssl_maximum_version,
|
| 1082 |
+
**self.conn_kw,
|
| 1083 |
+
)
|
| 1084 |
+
|
| 1085 |
+
def _validate_conn(self, conn: BaseHTTPConnection) -> None:
|
| 1086 |
+
"""
|
| 1087 |
+
Called right before a request is made, after the socket is created.
|
| 1088 |
+
"""
|
| 1089 |
+
super()._validate_conn(conn)
|
| 1090 |
+
|
| 1091 |
+
# Force connect early to allow us to validate the connection.
|
| 1092 |
+
if conn.is_closed:
|
| 1093 |
+
conn.connect()
|
| 1094 |
+
|
| 1095 |
+
# TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
|
| 1096 |
+
if not conn.is_verified and not conn.proxy_is_verified:
|
| 1097 |
+
warnings.warn(
|
| 1098 |
+
(
|
| 1099 |
+
f"Unverified HTTPS request is being made to host '{conn.host}'. "
|
| 1100 |
+
"Adding certificate verification is strongly advised. See: "
|
| 1101 |
+
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
|
| 1102 |
+
"#tls-warnings"
|
| 1103 |
+
),
|
| 1104 |
+
InsecureRequestWarning,
|
| 1105 |
+
)
|
| 1106 |
+
|
| 1107 |
+
|
| 1108 |
+
def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
|
| 1109 |
+
"""
|
| 1110 |
+
Given a url, return an :class:`.ConnectionPool` instance of its host.
|
| 1111 |
+
|
| 1112 |
+
This is a shortcut for not having to parse out the scheme, host, and port
|
| 1113 |
+
of the url before creating an :class:`.ConnectionPool` instance.
|
| 1114 |
+
|
| 1115 |
+
:param url:
|
| 1116 |
+
Absolute URL string that must include the scheme. Port is optional.
|
| 1117 |
+
|
| 1118 |
+
:param \\**kw:
|
| 1119 |
+
Passes additional parameters to the constructor of the appropriate
|
| 1120 |
+
:class:`.ConnectionPool`. Useful for specifying things like
|
| 1121 |
+
timeout, maxsize, headers, etc.
|
| 1122 |
+
|
| 1123 |
+
Example::
|
| 1124 |
+
|
| 1125 |
+
>>> conn = connection_from_url('http://google.com/')
|
| 1126 |
+
>>> r = conn.request('GET', '/')
|
| 1127 |
+
"""
|
| 1128 |
+
scheme, _, host, port, *_ = parse_url(url)
|
| 1129 |
+
scheme = scheme or "http"
|
| 1130 |
+
port = port or port_by_scheme.get(scheme, 80)
|
| 1131 |
+
if scheme == "https":
|
| 1132 |
+
return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
|
| 1133 |
+
else:
|
| 1134 |
+
return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
|
| 1135 |
+
|
| 1136 |
+
|
| 1137 |
+
@typing.overload
|
| 1138 |
+
def _normalize_host(host: None, scheme: str | None) -> None: ...
|
| 1139 |
+
|
| 1140 |
+
|
| 1141 |
+
@typing.overload
|
| 1142 |
+
def _normalize_host(host: str, scheme: str | None) -> str: ...
|
| 1143 |
+
|
| 1144 |
+
|
| 1145 |
+
def _normalize_host(host: str | None, scheme: str | None) -> str | None:
|
| 1146 |
+
"""
|
| 1147 |
+
Normalize hosts for comparisons and use with sockets.
|
| 1148 |
+
"""
|
| 1149 |
+
|
| 1150 |
+
host = normalize_host(host, scheme)
|
| 1151 |
+
|
| 1152 |
+
# httplib doesn't like it when we include brackets in IPv6 addresses
|
| 1153 |
+
# Specifically, if we include brackets but also pass the port then
|
| 1154 |
+
# httplib crazily doubles up the square brackets on the Host header.
|
| 1155 |
+
# Instead, we need to make sure we never pass ``None`` as the port.
|
| 1156 |
+
# However, for backward compatibility reasons we can't actually
|
| 1157 |
+
# *assert* that. See http://bugs.python.org/issue28539
|
| 1158 |
+
if host and host.startswith("[") and host.endswith("]"):
|
| 1159 |
+
host = host[1:-1]
|
| 1160 |
+
return host
|
| 1161 |
+
|
| 1162 |
+
|
| 1163 |
+
def _url_from_pool(
|
| 1164 |
+
pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
|
| 1165 |
+
) -> str:
|
| 1166 |
+
"""Returns the URL from a given connection pool. This is mainly used for testing and logging."""
|
| 1167 |
+
return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
|
| 1168 |
+
|
| 1169 |
+
|
| 1170 |
+
def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
|
| 1171 |
+
"""Drains a queue of connections and closes each one."""
|
| 1172 |
+
try:
|
| 1173 |
+
while True:
|
| 1174 |
+
conn = pool.get(block=False)
|
| 1175 |
+
if conn:
|
| 1176 |
+
conn.close()
|
| 1177 |
+
except queue.Empty:
|
| 1178 |
+
pass # Done.
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (192 Bytes). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/__pycache__/socks.cpython-312.pyc
ADDED
|
Binary file (8.17 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import urllib3.connection
|
| 4 |
+
|
| 5 |
+
from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
| 6 |
+
from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def inject_into_urllib3() -> None:
|
| 10 |
+
# override connection classes to use emscripten specific classes
|
| 11 |
+
# n.b. mypy complains about the overriding of classes below
|
| 12 |
+
# if it isn't ignored
|
| 13 |
+
HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
|
| 14 |
+
HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
|
| 15 |
+
urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment]
|
| 16 |
+
urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment]
|
| 17 |
+
urllib3.connection.VerifiedHTTPSConnection = EmscriptenHTTPSConnection # type: ignore[assignment]
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/connection.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import typing
|
| 5 |
+
|
| 6 |
+
# use http.client.HTTPException for consistency with non-emscripten
|
| 7 |
+
from http.client import HTTPException as HTTPException # noqa: F401
|
| 8 |
+
from http.client import ResponseNotReady
|
| 9 |
+
|
| 10 |
+
from ..._base_connection import _TYPE_BODY
|
| 11 |
+
from ...connection import HTTPConnection, ProxyConfig, port_by_scheme
|
| 12 |
+
from ...exceptions import TimeoutError
|
| 13 |
+
from ...response import BaseHTTPResponse
|
| 14 |
+
from ...util.connection import _TYPE_SOCKET_OPTIONS
|
| 15 |
+
from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
| 16 |
+
from ...util.url import Url
|
| 17 |
+
from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request
|
| 18 |
+
from .request import EmscriptenRequest
|
| 19 |
+
from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse
|
| 20 |
+
|
| 21 |
+
if typing.TYPE_CHECKING:
|
| 22 |
+
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class EmscriptenHTTPConnection:
|
| 26 |
+
default_port: typing.ClassVar[int] = port_by_scheme["http"]
|
| 27 |
+
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
| 28 |
+
|
| 29 |
+
timeout: None | (float)
|
| 30 |
+
|
| 31 |
+
host: str
|
| 32 |
+
port: int
|
| 33 |
+
blocksize: int
|
| 34 |
+
source_address: tuple[str, int] | None
|
| 35 |
+
socket_options: _TYPE_SOCKET_OPTIONS | None
|
| 36 |
+
|
| 37 |
+
proxy: Url | None
|
| 38 |
+
proxy_config: ProxyConfig | None
|
| 39 |
+
|
| 40 |
+
is_verified: bool = False
|
| 41 |
+
proxy_is_verified: bool | None = None
|
| 42 |
+
|
| 43 |
+
response_class: type[BaseHTTPResponse] = EmscriptenHttpResponseWrapper
|
| 44 |
+
_response: EmscriptenResponse | None
|
| 45 |
+
|
| 46 |
+
def __init__(
|
| 47 |
+
self,
|
| 48 |
+
host: str,
|
| 49 |
+
port: int = 0,
|
| 50 |
+
*,
|
| 51 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 52 |
+
source_address: tuple[str, int] | None = None,
|
| 53 |
+
blocksize: int = 8192,
|
| 54 |
+
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
|
| 55 |
+
proxy: Url | None = None,
|
| 56 |
+
proxy_config: ProxyConfig | None = None,
|
| 57 |
+
) -> None:
|
| 58 |
+
self.host = host
|
| 59 |
+
self.port = port
|
| 60 |
+
self.timeout = timeout if isinstance(timeout, float) else 0.0
|
| 61 |
+
self.scheme = "http"
|
| 62 |
+
self._closed = True
|
| 63 |
+
self._response = None
|
| 64 |
+
# ignore these things because we don't
|
| 65 |
+
# have control over that stuff
|
| 66 |
+
self.proxy = None
|
| 67 |
+
self.proxy_config = None
|
| 68 |
+
self.blocksize = blocksize
|
| 69 |
+
self.source_address = None
|
| 70 |
+
self.socket_options = None
|
| 71 |
+
self.is_verified = False
|
| 72 |
+
|
| 73 |
+
def set_tunnel(
|
| 74 |
+
self,
|
| 75 |
+
host: str,
|
| 76 |
+
port: int | None = 0,
|
| 77 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 78 |
+
scheme: str = "http",
|
| 79 |
+
) -> None:
|
| 80 |
+
pass
|
| 81 |
+
|
| 82 |
+
def connect(self) -> None:
|
| 83 |
+
pass
|
| 84 |
+
|
| 85 |
+
def request(
|
| 86 |
+
self,
|
| 87 |
+
method: str,
|
| 88 |
+
url: str,
|
| 89 |
+
body: _TYPE_BODY | None = None,
|
| 90 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 91 |
+
# We know *at least* botocore is depending on the order of the
|
| 92 |
+
# first 3 parameters so to be safe we only mark the later ones
|
| 93 |
+
# as keyword-only to ensure we have space to extend.
|
| 94 |
+
*,
|
| 95 |
+
chunked: bool = False,
|
| 96 |
+
preload_content: bool = True,
|
| 97 |
+
decode_content: bool = True,
|
| 98 |
+
enforce_content_length: bool = True,
|
| 99 |
+
) -> None:
|
| 100 |
+
self._closed = False
|
| 101 |
+
if url.startswith("/"):
|
| 102 |
+
if self.port is not None:
|
| 103 |
+
port = f":{self.port}"
|
| 104 |
+
else:
|
| 105 |
+
port = ""
|
| 106 |
+
# no scheme / host / port included, make a full url
|
| 107 |
+
url = f"{self.scheme}://{self.host}{port}{url}"
|
| 108 |
+
request = EmscriptenRequest(
|
| 109 |
+
url=url,
|
| 110 |
+
method=method,
|
| 111 |
+
timeout=self.timeout if self.timeout else 0,
|
| 112 |
+
decode_content=decode_content,
|
| 113 |
+
)
|
| 114 |
+
request.set_body(body)
|
| 115 |
+
if headers:
|
| 116 |
+
for k, v in headers.items():
|
| 117 |
+
request.set_header(k, v)
|
| 118 |
+
self._response = None
|
| 119 |
+
try:
|
| 120 |
+
if not preload_content:
|
| 121 |
+
self._response = send_streaming_request(request)
|
| 122 |
+
if self._response is None:
|
| 123 |
+
self._response = send_request(request)
|
| 124 |
+
except _TimeoutError as e:
|
| 125 |
+
raise TimeoutError(e.message) from e
|
| 126 |
+
except _RequestError as e:
|
| 127 |
+
raise HTTPException(e.message) from e
|
| 128 |
+
|
| 129 |
+
def getresponse(self) -> BaseHTTPResponse:
|
| 130 |
+
if self._response is not None:
|
| 131 |
+
return EmscriptenHttpResponseWrapper(
|
| 132 |
+
internal_response=self._response,
|
| 133 |
+
url=self._response.request.url,
|
| 134 |
+
connection=self,
|
| 135 |
+
)
|
| 136 |
+
else:
|
| 137 |
+
raise ResponseNotReady()
|
| 138 |
+
|
| 139 |
+
def close(self) -> None:
|
| 140 |
+
self._closed = True
|
| 141 |
+
self._response = None
|
| 142 |
+
|
| 143 |
+
@property
|
| 144 |
+
def is_closed(self) -> bool:
|
| 145 |
+
"""Whether the connection either is brand new or has been previously closed.
|
| 146 |
+
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
|
| 147 |
+
properties must be False.
|
| 148 |
+
"""
|
| 149 |
+
return self._closed
|
| 150 |
+
|
| 151 |
+
@property
|
| 152 |
+
def is_connected(self) -> bool:
|
| 153 |
+
"""Whether the connection is actively connected to any origin (proxy or target)"""
|
| 154 |
+
return True
|
| 155 |
+
|
| 156 |
+
@property
|
| 157 |
+
def has_connected_to_proxy(self) -> bool:
|
| 158 |
+
"""Whether the connection has successfully connected to its proxy.
|
| 159 |
+
This returns False if no proxy is in use. Used to determine whether
|
| 160 |
+
errors are coming from the proxy layer or from tunnelling to the target origin.
|
| 161 |
+
"""
|
| 162 |
+
return False
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class EmscriptenHTTPSConnection(EmscriptenHTTPConnection):
|
| 166 |
+
default_port = port_by_scheme["https"]
|
| 167 |
+
# all this is basically ignored, as browser handles https
|
| 168 |
+
cert_reqs: int | str | None = None
|
| 169 |
+
ca_certs: str | None = None
|
| 170 |
+
ca_cert_dir: str | None = None
|
| 171 |
+
ca_cert_data: None | str | bytes = None
|
| 172 |
+
cert_file: str | None
|
| 173 |
+
key_file: str | None
|
| 174 |
+
key_password: str | None
|
| 175 |
+
ssl_context: typing.Any | None
|
| 176 |
+
ssl_version: int | str | None = None
|
| 177 |
+
ssl_minimum_version: int | None = None
|
| 178 |
+
ssl_maximum_version: int | None = None
|
| 179 |
+
assert_hostname: None | str | typing.Literal[False]
|
| 180 |
+
assert_fingerprint: str | None = None
|
| 181 |
+
|
| 182 |
+
def __init__(
|
| 183 |
+
self,
|
| 184 |
+
host: str,
|
| 185 |
+
port: int = 0,
|
| 186 |
+
*,
|
| 187 |
+
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
| 188 |
+
source_address: tuple[str, int] | None = None,
|
| 189 |
+
blocksize: int = 16384,
|
| 190 |
+
socket_options: (
|
| 191 |
+
None | _TYPE_SOCKET_OPTIONS
|
| 192 |
+
) = HTTPConnection.default_socket_options,
|
| 193 |
+
proxy: Url | None = None,
|
| 194 |
+
proxy_config: ProxyConfig | None = None,
|
| 195 |
+
cert_reqs: int | str | None = None,
|
| 196 |
+
assert_hostname: None | str | typing.Literal[False] = None,
|
| 197 |
+
assert_fingerprint: str | None = None,
|
| 198 |
+
server_hostname: str | None = None,
|
| 199 |
+
ssl_context: typing.Any | None = None,
|
| 200 |
+
ca_certs: str | None = None,
|
| 201 |
+
ca_cert_dir: str | None = None,
|
| 202 |
+
ca_cert_data: None | str | bytes = None,
|
| 203 |
+
ssl_minimum_version: int | None = None,
|
| 204 |
+
ssl_maximum_version: int | None = None,
|
| 205 |
+
ssl_version: int | str | None = None, # Deprecated
|
| 206 |
+
cert_file: str | None = None,
|
| 207 |
+
key_file: str | None = None,
|
| 208 |
+
key_password: str | None = None,
|
| 209 |
+
) -> None:
|
| 210 |
+
super().__init__(
|
| 211 |
+
host,
|
| 212 |
+
port=port,
|
| 213 |
+
timeout=timeout,
|
| 214 |
+
source_address=source_address,
|
| 215 |
+
blocksize=blocksize,
|
| 216 |
+
socket_options=socket_options,
|
| 217 |
+
proxy=proxy,
|
| 218 |
+
proxy_config=proxy_config,
|
| 219 |
+
)
|
| 220 |
+
self.scheme = "https"
|
| 221 |
+
|
| 222 |
+
self.key_file = key_file
|
| 223 |
+
self.cert_file = cert_file
|
| 224 |
+
self.key_password = key_password
|
| 225 |
+
self.ssl_context = ssl_context
|
| 226 |
+
self.server_hostname = server_hostname
|
| 227 |
+
self.assert_hostname = assert_hostname
|
| 228 |
+
self.assert_fingerprint = assert_fingerprint
|
| 229 |
+
self.ssl_version = ssl_version
|
| 230 |
+
self.ssl_minimum_version = ssl_minimum_version
|
| 231 |
+
self.ssl_maximum_version = ssl_maximum_version
|
| 232 |
+
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
| 233 |
+
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
| 234 |
+
self.ca_cert_data = ca_cert_data
|
| 235 |
+
|
| 236 |
+
self.cert_reqs = None
|
| 237 |
+
|
| 238 |
+
# The browser will automatically verify all requests.
|
| 239 |
+
# We have no control over that setting.
|
| 240 |
+
self.is_verified = True
|
| 241 |
+
|
| 242 |
+
def set_cert(
|
| 243 |
+
self,
|
| 244 |
+
key_file: str | None = None,
|
| 245 |
+
cert_file: str | None = None,
|
| 246 |
+
cert_reqs: int | str | None = None,
|
| 247 |
+
key_password: str | None = None,
|
| 248 |
+
ca_certs: str | None = None,
|
| 249 |
+
assert_hostname: None | str | typing.Literal[False] = None,
|
| 250 |
+
assert_fingerprint: str | None = None,
|
| 251 |
+
ca_cert_dir: str | None = None,
|
| 252 |
+
ca_cert_data: None | str | bytes = None,
|
| 253 |
+
) -> None:
|
| 254 |
+
pass
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
# verify that this class implements BaseHTTP(s) connection correctly
|
| 258 |
+
if typing.TYPE_CHECKING:
|
| 259 |
+
_supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0)
|
| 260 |
+
_supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0)
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/emscripten_fetch_worker.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
let Status = {
|
| 2 |
+
SUCCESS_HEADER: -1,
|
| 3 |
+
SUCCESS_EOF: -2,
|
| 4 |
+
ERROR_TIMEOUT: -3,
|
| 5 |
+
ERROR_EXCEPTION: -4,
|
| 6 |
+
};
|
| 7 |
+
|
| 8 |
+
let connections = new Map();
|
| 9 |
+
let nextConnectionID = 1;
|
| 10 |
+
const encoder = new TextEncoder();
|
| 11 |
+
|
| 12 |
+
self.addEventListener("message", async function (event) {
|
| 13 |
+
if (event.data.close) {
|
| 14 |
+
let connectionID = event.data.close;
|
| 15 |
+
connections.delete(connectionID);
|
| 16 |
+
return;
|
| 17 |
+
} else if (event.data.getMore) {
|
| 18 |
+
let connectionID = event.data.getMore;
|
| 19 |
+
let { curOffset, value, reader, intBuffer, byteBuffer } =
|
| 20 |
+
connections.get(connectionID);
|
| 21 |
+
// if we still have some in buffer, then just send it back straight away
|
| 22 |
+
if (!value || curOffset >= value.length) {
|
| 23 |
+
// read another buffer if required
|
| 24 |
+
try {
|
| 25 |
+
let readResponse = await reader.read();
|
| 26 |
+
|
| 27 |
+
if (readResponse.done) {
|
| 28 |
+
// read everything - clear connection and return
|
| 29 |
+
connections.delete(connectionID);
|
| 30 |
+
Atomics.store(intBuffer, 0, Status.SUCCESS_EOF);
|
| 31 |
+
Atomics.notify(intBuffer, 0);
|
| 32 |
+
// finished reading successfully
|
| 33 |
+
// return from event handler
|
| 34 |
+
return;
|
| 35 |
+
}
|
| 36 |
+
curOffset = 0;
|
| 37 |
+
connections.get(connectionID).value = readResponse.value;
|
| 38 |
+
value = readResponse.value;
|
| 39 |
+
} catch (error) {
|
| 40 |
+
console.log("Request exception:", error);
|
| 41 |
+
let errorBytes = encoder.encode(error.message);
|
| 42 |
+
let written = errorBytes.length;
|
| 43 |
+
byteBuffer.set(errorBytes);
|
| 44 |
+
intBuffer[1] = written;
|
| 45 |
+
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
|
| 46 |
+
Atomics.notify(intBuffer, 0);
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
// send as much buffer as we can
|
| 51 |
+
let curLen = value.length - curOffset;
|
| 52 |
+
if (curLen > byteBuffer.length) {
|
| 53 |
+
curLen = byteBuffer.length;
|
| 54 |
+
}
|
| 55 |
+
byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0);
|
| 56 |
+
|
| 57 |
+
Atomics.store(intBuffer, 0, curLen); // store current length in bytes
|
| 58 |
+
Atomics.notify(intBuffer, 0);
|
| 59 |
+
curOffset += curLen;
|
| 60 |
+
connections.get(connectionID).curOffset = curOffset;
|
| 61 |
+
|
| 62 |
+
return;
|
| 63 |
+
} else {
|
| 64 |
+
// start fetch
|
| 65 |
+
let connectionID = nextConnectionID;
|
| 66 |
+
nextConnectionID += 1;
|
| 67 |
+
const intBuffer = new Int32Array(event.data.buffer);
|
| 68 |
+
const byteBuffer = new Uint8Array(event.data.buffer, 8);
|
| 69 |
+
try {
|
| 70 |
+
const response = await fetch(event.data.url, event.data.fetchParams);
|
| 71 |
+
// return the headers first via textencoder
|
| 72 |
+
var headers = [];
|
| 73 |
+
for (const pair of response.headers.entries()) {
|
| 74 |
+
headers.push([pair[0], pair[1]]);
|
| 75 |
+
}
|
| 76 |
+
let headerObj = {
|
| 77 |
+
headers: headers,
|
| 78 |
+
status: response.status,
|
| 79 |
+
connectionID,
|
| 80 |
+
};
|
| 81 |
+
const headerText = JSON.stringify(headerObj);
|
| 82 |
+
let headerBytes = encoder.encode(headerText);
|
| 83 |
+
let written = headerBytes.length;
|
| 84 |
+
byteBuffer.set(headerBytes);
|
| 85 |
+
intBuffer[1] = written;
|
| 86 |
+
// make a connection
|
| 87 |
+
connections.set(connectionID, {
|
| 88 |
+
reader: response.body.getReader(),
|
| 89 |
+
intBuffer: intBuffer,
|
| 90 |
+
byteBuffer: byteBuffer,
|
| 91 |
+
value: undefined,
|
| 92 |
+
curOffset: 0,
|
| 93 |
+
});
|
| 94 |
+
// set header ready
|
| 95 |
+
Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER);
|
| 96 |
+
Atomics.notify(intBuffer, 0);
|
| 97 |
+
// all fetching after this goes through a new postmessage call with getMore
|
| 98 |
+
// this allows for parallel requests
|
| 99 |
+
} catch (error) {
|
| 100 |
+
console.log("Request exception:", error);
|
| 101 |
+
let errorBytes = encoder.encode(error.message);
|
| 102 |
+
let written = errorBytes.length;
|
| 103 |
+
byteBuffer.set(errorBytes);
|
| 104 |
+
intBuffer[1] = written;
|
| 105 |
+
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
|
| 106 |
+
Atomics.notify(intBuffer, 0);
|
| 107 |
+
}
|
| 108 |
+
}
|
| 109 |
+
});
|
| 110 |
+
self.postMessage({ inited: true });
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/fetch.py
ADDED
|
@@ -0,0 +1,726 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Support for streaming http requests in emscripten.
|
| 3 |
+
|
| 4 |
+
A few caveats -
|
| 5 |
+
|
| 6 |
+
If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled
|
| 7 |
+
https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md
|
| 8 |
+
*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the
|
| 9 |
+
JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case
|
| 10 |
+
timeouts and streaming should just work.
|
| 11 |
+
|
| 12 |
+
Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming.
|
| 13 |
+
|
| 14 |
+
This approach has several caveats:
|
| 15 |
+
|
| 16 |
+
Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
|
| 17 |
+
Streaming only works if you're running pyodide in a web worker.
|
| 18 |
+
|
| 19 |
+
Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
|
| 20 |
+
operation, so it requires that you have crossOriginIsolation enabled, by serving over https
|
| 21 |
+
(or from localhost) with the two headers below set:
|
| 22 |
+
|
| 23 |
+
Cross-Origin-Opener-Policy: same-origin
|
| 24 |
+
Cross-Origin-Embedder-Policy: require-corp
|
| 25 |
+
|
| 26 |
+
You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
|
| 27 |
+
JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
|
| 28 |
+
request into a buffer and then returning it. it shows a warning in the JavaScript console in this case.
|
| 29 |
+
|
| 30 |
+
Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
|
| 31 |
+
control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
|
| 32 |
+
|
| 33 |
+
NB: in this code, there are a lot of JavaScript objects. They are named js_*
|
| 34 |
+
to make it clear what type of object they are.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
from __future__ import annotations
|
| 38 |
+
|
| 39 |
+
import io
|
| 40 |
+
import json
|
| 41 |
+
from email.parser import Parser
|
| 42 |
+
from importlib.resources import files
|
| 43 |
+
from typing import TYPE_CHECKING, Any
|
| 44 |
+
|
| 45 |
+
import js # type: ignore[import-not-found]
|
| 46 |
+
from pyodide.ffi import ( # type: ignore[import-not-found]
|
| 47 |
+
JsArray,
|
| 48 |
+
JsException,
|
| 49 |
+
JsProxy,
|
| 50 |
+
to_js,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
if TYPE_CHECKING:
|
| 54 |
+
from typing_extensions import Buffer
|
| 55 |
+
|
| 56 |
+
from .request import EmscriptenRequest
|
| 57 |
+
from .response import EmscriptenResponse
|
| 58 |
+
|
| 59 |
+
"""
|
| 60 |
+
There are some headers that trigger unintended CORS preflight requests.
|
| 61 |
+
See also https://github.com/koenvo/pyodide-http/issues/22
|
| 62 |
+
"""
|
| 63 |
+
HEADERS_TO_IGNORE = ("user-agent",)
|
| 64 |
+
|
| 65 |
+
SUCCESS_HEADER = -1
|
| 66 |
+
SUCCESS_EOF = -2
|
| 67 |
+
ERROR_TIMEOUT = -3
|
| 68 |
+
ERROR_EXCEPTION = -4
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class _RequestError(Exception):
|
| 72 |
+
def __init__(
|
| 73 |
+
self,
|
| 74 |
+
message: str | None = None,
|
| 75 |
+
*,
|
| 76 |
+
request: EmscriptenRequest | None = None,
|
| 77 |
+
response: EmscriptenResponse | None = None,
|
| 78 |
+
):
|
| 79 |
+
self.request = request
|
| 80 |
+
self.response = response
|
| 81 |
+
self.message = message
|
| 82 |
+
super().__init__(self.message)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class _StreamingError(_RequestError):
|
| 86 |
+
pass
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class _TimeoutError(_RequestError):
|
| 90 |
+
pass
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy:
|
| 94 |
+
return to_js(dict_val, dict_converter=js.Object.fromEntries)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class _ReadStream(io.RawIOBase):
|
| 98 |
+
def __init__(
|
| 99 |
+
self,
|
| 100 |
+
int_buffer: JsArray,
|
| 101 |
+
byte_buffer: JsArray,
|
| 102 |
+
timeout: float,
|
| 103 |
+
worker: JsProxy,
|
| 104 |
+
connection_id: int,
|
| 105 |
+
request: EmscriptenRequest,
|
| 106 |
+
):
|
| 107 |
+
self.int_buffer = int_buffer
|
| 108 |
+
self.byte_buffer = byte_buffer
|
| 109 |
+
self.read_pos = 0
|
| 110 |
+
self.read_len = 0
|
| 111 |
+
self.connection_id = connection_id
|
| 112 |
+
self.worker = worker
|
| 113 |
+
self.timeout = int(1000 * timeout) if timeout > 0 else None
|
| 114 |
+
self.is_live = True
|
| 115 |
+
self._is_closed = False
|
| 116 |
+
self.request: EmscriptenRequest | None = request
|
| 117 |
+
|
| 118 |
+
def __del__(self) -> None:
|
| 119 |
+
self.close()
|
| 120 |
+
|
| 121 |
+
# this is compatible with _base_connection
|
| 122 |
+
def is_closed(self) -> bool:
|
| 123 |
+
return self._is_closed
|
| 124 |
+
|
| 125 |
+
# for compatibility with RawIOBase
|
| 126 |
+
@property
|
| 127 |
+
def closed(self) -> bool:
|
| 128 |
+
return self.is_closed()
|
| 129 |
+
|
| 130 |
+
def close(self) -> None:
|
| 131 |
+
if self.is_closed():
|
| 132 |
+
return
|
| 133 |
+
self.read_len = 0
|
| 134 |
+
self.read_pos = 0
|
| 135 |
+
self.int_buffer = None
|
| 136 |
+
self.byte_buffer = None
|
| 137 |
+
self._is_closed = True
|
| 138 |
+
self.request = None
|
| 139 |
+
if self.is_live:
|
| 140 |
+
self.worker.postMessage(_obj_from_dict({"close": self.connection_id}))
|
| 141 |
+
self.is_live = False
|
| 142 |
+
super().close()
|
| 143 |
+
|
| 144 |
+
def readable(self) -> bool:
|
| 145 |
+
return True
|
| 146 |
+
|
| 147 |
+
def writable(self) -> bool:
|
| 148 |
+
return False
|
| 149 |
+
|
| 150 |
+
def seekable(self) -> bool:
|
| 151 |
+
return False
|
| 152 |
+
|
| 153 |
+
def readinto(self, byte_obj: Buffer) -> int:
|
| 154 |
+
if not self.int_buffer:
|
| 155 |
+
raise _StreamingError(
|
| 156 |
+
"No buffer for stream in _ReadStream.readinto",
|
| 157 |
+
request=self.request,
|
| 158 |
+
response=None,
|
| 159 |
+
)
|
| 160 |
+
if self.read_len == 0:
|
| 161 |
+
# wait for the worker to send something
|
| 162 |
+
js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT)
|
| 163 |
+
self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id}))
|
| 164 |
+
if (
|
| 165 |
+
js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout)
|
| 166 |
+
== "timed-out"
|
| 167 |
+
):
|
| 168 |
+
raise _TimeoutError
|
| 169 |
+
data_len = self.int_buffer[0]
|
| 170 |
+
if data_len > 0:
|
| 171 |
+
self.read_len = data_len
|
| 172 |
+
self.read_pos = 0
|
| 173 |
+
elif data_len == ERROR_EXCEPTION:
|
| 174 |
+
string_len = self.int_buffer[1]
|
| 175 |
+
# decode the error string
|
| 176 |
+
js_decoder = js.TextDecoder.new()
|
| 177 |
+
json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len))
|
| 178 |
+
raise _StreamingError(
|
| 179 |
+
f"Exception thrown in fetch: {json_str}",
|
| 180 |
+
request=self.request,
|
| 181 |
+
response=None,
|
| 182 |
+
)
|
| 183 |
+
else:
|
| 184 |
+
# EOF, free the buffers and return zero
|
| 185 |
+
# and free the request
|
| 186 |
+
self.is_live = False
|
| 187 |
+
self.close()
|
| 188 |
+
return 0
|
| 189 |
+
# copy from int32array to python bytes
|
| 190 |
+
ret_length = min(self.read_len, len(memoryview(byte_obj)))
|
| 191 |
+
subarray = self.byte_buffer.subarray(
|
| 192 |
+
self.read_pos, self.read_pos + ret_length
|
| 193 |
+
).to_py()
|
| 194 |
+
memoryview(byte_obj)[0:ret_length] = subarray
|
| 195 |
+
self.read_len -= ret_length
|
| 196 |
+
self.read_pos += ret_length
|
| 197 |
+
return ret_length
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
class _StreamingFetcher:
|
| 201 |
+
def __init__(self) -> None:
|
| 202 |
+
# make web-worker and data buffer on startup
|
| 203 |
+
self.streaming_ready = False
|
| 204 |
+
streaming_worker_code = (
|
| 205 |
+
files(__package__)
|
| 206 |
+
.joinpath("emscripten_fetch_worker.js")
|
| 207 |
+
.read_text(encoding="utf-8")
|
| 208 |
+
)
|
| 209 |
+
js_data_blob = js.Blob.new(
|
| 210 |
+
to_js([streaming_worker_code], create_pyproxies=False),
|
| 211 |
+
_obj_from_dict({"type": "application/javascript"}),
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None:
|
| 215 |
+
def onMsg(e: JsProxy) -> None:
|
| 216 |
+
self.streaming_ready = True
|
| 217 |
+
js_resolve_fn(e)
|
| 218 |
+
|
| 219 |
+
def onErr(e: JsProxy) -> None:
|
| 220 |
+
js_reject_fn(e) # Defensive: never happens in ci
|
| 221 |
+
|
| 222 |
+
self.js_worker.onmessage = onMsg
|
| 223 |
+
self.js_worker.onerror = onErr
|
| 224 |
+
|
| 225 |
+
js_data_url = js.URL.createObjectURL(js_data_blob)
|
| 226 |
+
self.js_worker = js.globalThis.Worker.new(js_data_url)
|
| 227 |
+
self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver)
|
| 228 |
+
|
| 229 |
+
def send(self, request: EmscriptenRequest) -> EmscriptenResponse:
|
| 230 |
+
headers = {
|
| 231 |
+
k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
body = request.body
|
| 235 |
+
fetch_data = {"headers": headers, "body": to_js(body), "method": request.method}
|
| 236 |
+
# start the request off in the worker
|
| 237 |
+
timeout = int(1000 * request.timeout) if request.timeout > 0 else None
|
| 238 |
+
js_shared_buffer = js.SharedArrayBuffer.new(1048576)
|
| 239 |
+
js_int_buffer = js.Int32Array.new(js_shared_buffer)
|
| 240 |
+
js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8)
|
| 241 |
+
|
| 242 |
+
js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT)
|
| 243 |
+
js.Atomics.notify(js_int_buffer, 0)
|
| 244 |
+
js_absolute_url = js.URL.new(request.url, js.location).href
|
| 245 |
+
self.js_worker.postMessage(
|
| 246 |
+
_obj_from_dict(
|
| 247 |
+
{
|
| 248 |
+
"buffer": js_shared_buffer,
|
| 249 |
+
"url": js_absolute_url,
|
| 250 |
+
"fetchParams": fetch_data,
|
| 251 |
+
}
|
| 252 |
+
)
|
| 253 |
+
)
|
| 254 |
+
# wait for the worker to send something
|
| 255 |
+
js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout)
|
| 256 |
+
if js_int_buffer[0] == ERROR_TIMEOUT:
|
| 257 |
+
raise _TimeoutError(
|
| 258 |
+
"Timeout connecting to streaming request",
|
| 259 |
+
request=request,
|
| 260 |
+
response=None,
|
| 261 |
+
)
|
| 262 |
+
elif js_int_buffer[0] == SUCCESS_HEADER:
|
| 263 |
+
# got response
|
| 264 |
+
# header length is in second int of intBuffer
|
| 265 |
+
string_len = js_int_buffer[1]
|
| 266 |
+
# decode the rest to a JSON string
|
| 267 |
+
js_decoder = js.TextDecoder.new()
|
| 268 |
+
# this does a copy (the slice) because decode can't work on shared array
|
| 269 |
+
# for some silly reason
|
| 270 |
+
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
|
| 271 |
+
# get it as an object
|
| 272 |
+
response_obj = json.loads(json_str)
|
| 273 |
+
return EmscriptenResponse(
|
| 274 |
+
request=request,
|
| 275 |
+
status_code=response_obj["status"],
|
| 276 |
+
headers=response_obj["headers"],
|
| 277 |
+
body=_ReadStream(
|
| 278 |
+
js_int_buffer,
|
| 279 |
+
js_byte_buffer,
|
| 280 |
+
request.timeout,
|
| 281 |
+
self.js_worker,
|
| 282 |
+
response_obj["connectionID"],
|
| 283 |
+
request,
|
| 284 |
+
),
|
| 285 |
+
)
|
| 286 |
+
elif js_int_buffer[0] == ERROR_EXCEPTION:
|
| 287 |
+
string_len = js_int_buffer[1]
|
| 288 |
+
# decode the error string
|
| 289 |
+
js_decoder = js.TextDecoder.new()
|
| 290 |
+
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
|
| 291 |
+
raise _StreamingError(
|
| 292 |
+
f"Exception thrown in fetch: {json_str}", request=request, response=None
|
| 293 |
+
)
|
| 294 |
+
else:
|
| 295 |
+
raise _StreamingError(
|
| 296 |
+
f"Unknown status from worker in fetch: {js_int_buffer[0]}",
|
| 297 |
+
request=request,
|
| 298 |
+
response=None,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
class _JSPIReadStream(io.RawIOBase):
|
| 303 |
+
"""
|
| 304 |
+
A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch
|
| 305 |
+
response. This requires support for WebAssembly JavaScript Promise Integration
|
| 306 |
+
in the containing browser, and for pyodide to be launched via runPythonAsync.
|
| 307 |
+
|
| 308 |
+
:param js_read_stream:
|
| 309 |
+
The JavaScript stream reader
|
| 310 |
+
|
| 311 |
+
:param timeout:
|
| 312 |
+
Timeout in seconds
|
| 313 |
+
|
| 314 |
+
:param request:
|
| 315 |
+
The request we're handling
|
| 316 |
+
|
| 317 |
+
:param response:
|
| 318 |
+
The response this stream relates to
|
| 319 |
+
|
| 320 |
+
:param js_abort_controller:
|
| 321 |
+
A JavaScript AbortController object, used for timeouts
|
| 322 |
+
"""
|
| 323 |
+
|
| 324 |
+
def __init__(
|
| 325 |
+
self,
|
| 326 |
+
js_read_stream: Any,
|
| 327 |
+
timeout: float,
|
| 328 |
+
request: EmscriptenRequest,
|
| 329 |
+
response: EmscriptenResponse,
|
| 330 |
+
js_abort_controller: Any, # JavaScript AbortController for timeouts
|
| 331 |
+
):
|
| 332 |
+
self.js_read_stream = js_read_stream
|
| 333 |
+
self.timeout = timeout
|
| 334 |
+
self._is_closed = False
|
| 335 |
+
self._is_done = False
|
| 336 |
+
self.request: EmscriptenRequest | None = request
|
| 337 |
+
self.response: EmscriptenResponse | None = response
|
| 338 |
+
self.current_buffer = None
|
| 339 |
+
self.current_buffer_pos = 0
|
| 340 |
+
self.js_abort_controller = js_abort_controller
|
| 341 |
+
|
| 342 |
+
def __del__(self) -> None:
|
| 343 |
+
self.close()
|
| 344 |
+
|
| 345 |
+
# this is compatible with _base_connection
|
| 346 |
+
def is_closed(self) -> bool:
|
| 347 |
+
return self._is_closed
|
| 348 |
+
|
| 349 |
+
# for compatibility with RawIOBase
|
| 350 |
+
@property
|
| 351 |
+
def closed(self) -> bool:
|
| 352 |
+
return self.is_closed()
|
| 353 |
+
|
| 354 |
+
def close(self) -> None:
|
| 355 |
+
if self.is_closed():
|
| 356 |
+
return
|
| 357 |
+
self.read_len = 0
|
| 358 |
+
self.read_pos = 0
|
| 359 |
+
self.js_read_stream.cancel()
|
| 360 |
+
self.js_read_stream = None
|
| 361 |
+
self._is_closed = True
|
| 362 |
+
self._is_done = True
|
| 363 |
+
self.request = None
|
| 364 |
+
self.response = None
|
| 365 |
+
super().close()
|
| 366 |
+
|
| 367 |
+
def readable(self) -> bool:
|
| 368 |
+
return True
|
| 369 |
+
|
| 370 |
+
def writable(self) -> bool:
|
| 371 |
+
return False
|
| 372 |
+
|
| 373 |
+
def seekable(self) -> bool:
|
| 374 |
+
return False
|
| 375 |
+
|
| 376 |
+
def _get_next_buffer(self) -> bool:
|
| 377 |
+
result_js = _run_sync_with_timeout(
|
| 378 |
+
self.js_read_stream.read(),
|
| 379 |
+
self.timeout,
|
| 380 |
+
self.js_abort_controller,
|
| 381 |
+
request=self.request,
|
| 382 |
+
response=self.response,
|
| 383 |
+
)
|
| 384 |
+
if result_js.done:
|
| 385 |
+
self._is_done = True
|
| 386 |
+
return False
|
| 387 |
+
else:
|
| 388 |
+
self.current_buffer = result_js.value.to_py()
|
| 389 |
+
self.current_buffer_pos = 0
|
| 390 |
+
return True
|
| 391 |
+
|
| 392 |
+
def readinto(self, byte_obj: Buffer) -> int:
|
| 393 |
+
if self.current_buffer is None:
|
| 394 |
+
if not self._get_next_buffer() or self.current_buffer is None:
|
| 395 |
+
self.close()
|
| 396 |
+
return 0
|
| 397 |
+
ret_length = min(
|
| 398 |
+
len(byte_obj), len(self.current_buffer) - self.current_buffer_pos
|
| 399 |
+
)
|
| 400 |
+
byte_obj[0:ret_length] = self.current_buffer[
|
| 401 |
+
self.current_buffer_pos : self.current_buffer_pos + ret_length
|
| 402 |
+
]
|
| 403 |
+
self.current_buffer_pos += ret_length
|
| 404 |
+
if self.current_buffer_pos == len(self.current_buffer):
|
| 405 |
+
self.current_buffer = None
|
| 406 |
+
return ret_length
|
| 407 |
+
|
| 408 |
+
|
| 409 |
+
# check if we are in a worker or not
|
| 410 |
+
def is_in_browser_main_thread() -> bool:
|
| 411 |
+
return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window
|
| 412 |
+
|
| 413 |
+
|
| 414 |
+
def is_cross_origin_isolated() -> bool:
|
| 415 |
+
return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
def is_in_node() -> bool:
|
| 419 |
+
return (
|
| 420 |
+
hasattr(js, "process")
|
| 421 |
+
and hasattr(js.process, "release")
|
| 422 |
+
and hasattr(js.process.release, "name")
|
| 423 |
+
and js.process.release.name == "node"
|
| 424 |
+
)
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
def is_worker_available() -> bool:
|
| 428 |
+
return hasattr(js, "Worker") and hasattr(js, "Blob")
|
| 429 |
+
|
| 430 |
+
|
| 431 |
+
_fetcher: _StreamingFetcher | None = None
|
| 432 |
+
|
| 433 |
+
if is_worker_available() and (
|
| 434 |
+
(is_cross_origin_isolated() and not is_in_browser_main_thread())
|
| 435 |
+
and (not is_in_node())
|
| 436 |
+
):
|
| 437 |
+
_fetcher = _StreamingFetcher()
|
| 438 |
+
else:
|
| 439 |
+
_fetcher = None
|
| 440 |
+
|
| 441 |
+
|
| 442 |
+
NODE_JSPI_ERROR = (
|
| 443 |
+
"urllib3 only works in Node.js with pyodide.runPythonAsync"
|
| 444 |
+
" and requires the flag --experimental-wasm-stack-switching in "
|
| 445 |
+
" versions of node <24."
|
| 446 |
+
)
|
| 447 |
+
|
| 448 |
+
|
| 449 |
+
def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None:
|
| 450 |
+
if has_jspi():
|
| 451 |
+
return send_jspi_request(request, True)
|
| 452 |
+
elif is_in_node():
|
| 453 |
+
raise _RequestError(
|
| 454 |
+
message=NODE_JSPI_ERROR,
|
| 455 |
+
request=request,
|
| 456 |
+
response=None,
|
| 457 |
+
)
|
| 458 |
+
|
| 459 |
+
if _fetcher and streaming_ready():
|
| 460 |
+
return _fetcher.send(request)
|
| 461 |
+
else:
|
| 462 |
+
_show_streaming_warning()
|
| 463 |
+
return None
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
_SHOWN_TIMEOUT_WARNING = False
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def _show_timeout_warning() -> None:
|
| 470 |
+
global _SHOWN_TIMEOUT_WARNING
|
| 471 |
+
if not _SHOWN_TIMEOUT_WARNING:
|
| 472 |
+
_SHOWN_TIMEOUT_WARNING = True
|
| 473 |
+
message = "Warning: Timeout is not available on main browser thread"
|
| 474 |
+
js.console.warn(message)
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
_SHOWN_STREAMING_WARNING = False
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
def _show_streaming_warning() -> None:
|
| 481 |
+
global _SHOWN_STREAMING_WARNING
|
| 482 |
+
if not _SHOWN_STREAMING_WARNING:
|
| 483 |
+
_SHOWN_STREAMING_WARNING = True
|
| 484 |
+
message = "Can't stream HTTP requests because: \n"
|
| 485 |
+
if not is_cross_origin_isolated():
|
| 486 |
+
message += " Page is not cross-origin isolated\n"
|
| 487 |
+
if is_in_browser_main_thread():
|
| 488 |
+
message += " Python is running in main browser thread\n"
|
| 489 |
+
if not is_worker_available():
|
| 490 |
+
message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in
|
| 491 |
+
if streaming_ready() is False:
|
| 492 |
+
message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
|
| 493 |
+
is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`"""
|
| 494 |
+
from js import console
|
| 495 |
+
|
| 496 |
+
console.warn(message)
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
def send_request(request: EmscriptenRequest) -> EmscriptenResponse:
|
| 500 |
+
if has_jspi():
|
| 501 |
+
return send_jspi_request(request, False)
|
| 502 |
+
elif is_in_node():
|
| 503 |
+
raise _RequestError(
|
| 504 |
+
message=NODE_JSPI_ERROR,
|
| 505 |
+
request=request,
|
| 506 |
+
response=None,
|
| 507 |
+
)
|
| 508 |
+
try:
|
| 509 |
+
js_xhr = js.XMLHttpRequest.new()
|
| 510 |
+
|
| 511 |
+
if not is_in_browser_main_thread():
|
| 512 |
+
js_xhr.responseType = "arraybuffer"
|
| 513 |
+
if request.timeout:
|
| 514 |
+
js_xhr.timeout = int(request.timeout * 1000)
|
| 515 |
+
else:
|
| 516 |
+
js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15")
|
| 517 |
+
if request.timeout:
|
| 518 |
+
# timeout isn't available on the main thread - show a warning in console
|
| 519 |
+
# if it is set
|
| 520 |
+
_show_timeout_warning()
|
| 521 |
+
|
| 522 |
+
js_xhr.open(request.method, request.url, False)
|
| 523 |
+
for name, value in request.headers.items():
|
| 524 |
+
if name.lower() not in HEADERS_TO_IGNORE:
|
| 525 |
+
js_xhr.setRequestHeader(name, value)
|
| 526 |
+
|
| 527 |
+
js_xhr.send(to_js(request.body))
|
| 528 |
+
|
| 529 |
+
headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders()))
|
| 530 |
+
|
| 531 |
+
if not is_in_browser_main_thread():
|
| 532 |
+
body = js_xhr.response.to_py().tobytes()
|
| 533 |
+
else:
|
| 534 |
+
body = js_xhr.response.encode("ISO-8859-15")
|
| 535 |
+
return EmscriptenResponse(
|
| 536 |
+
status_code=js_xhr.status, headers=headers, body=body, request=request
|
| 537 |
+
)
|
| 538 |
+
except JsException as err:
|
| 539 |
+
if err.name == "TimeoutError":
|
| 540 |
+
raise _TimeoutError(err.message, request=request)
|
| 541 |
+
elif err.name == "NetworkError":
|
| 542 |
+
raise _RequestError(err.message, request=request)
|
| 543 |
+
else:
|
| 544 |
+
# general http error
|
| 545 |
+
raise _RequestError(err.message, request=request)
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def send_jspi_request(
|
| 549 |
+
request: EmscriptenRequest, streaming: bool
|
| 550 |
+
) -> EmscriptenResponse:
|
| 551 |
+
"""
|
| 552 |
+
Send a request using WebAssembly JavaScript Promise Integration
|
| 553 |
+
to wrap the asynchronous JavaScript fetch api (experimental).
|
| 554 |
+
|
| 555 |
+
:param request:
|
| 556 |
+
Request to send
|
| 557 |
+
|
| 558 |
+
:param streaming:
|
| 559 |
+
Whether to stream the response
|
| 560 |
+
|
| 561 |
+
:return: The response object
|
| 562 |
+
:rtype: EmscriptenResponse
|
| 563 |
+
"""
|
| 564 |
+
timeout = request.timeout
|
| 565 |
+
js_abort_controller = js.AbortController.new()
|
| 566 |
+
headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE}
|
| 567 |
+
req_body = request.body
|
| 568 |
+
fetch_data = {
|
| 569 |
+
"headers": headers,
|
| 570 |
+
"body": to_js(req_body),
|
| 571 |
+
"method": request.method,
|
| 572 |
+
"signal": js_abort_controller.signal,
|
| 573 |
+
}
|
| 574 |
+
# Node.js returns the whole response (unlike opaqueredirect in browsers),
|
| 575 |
+
# so urllib3 can set `redirect: manual` to control redirects itself.
|
| 576 |
+
# https://stackoverflow.com/a/78524615
|
| 577 |
+
if _is_node_js():
|
| 578 |
+
fetch_data["redirect"] = "manual"
|
| 579 |
+
# Call JavaScript fetch (async api, returns a promise)
|
| 580 |
+
fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data))
|
| 581 |
+
# Now suspend WebAssembly until we resolve that promise
|
| 582 |
+
# or time out.
|
| 583 |
+
response_js = _run_sync_with_timeout(
|
| 584 |
+
fetcher_promise_js,
|
| 585 |
+
timeout,
|
| 586 |
+
js_abort_controller,
|
| 587 |
+
request=request,
|
| 588 |
+
response=None,
|
| 589 |
+
)
|
| 590 |
+
headers = {}
|
| 591 |
+
header_iter = response_js.headers.entries()
|
| 592 |
+
while True:
|
| 593 |
+
iter_value_js = header_iter.next()
|
| 594 |
+
if getattr(iter_value_js, "done", False):
|
| 595 |
+
break
|
| 596 |
+
else:
|
| 597 |
+
headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1])
|
| 598 |
+
status_code = response_js.status
|
| 599 |
+
body: bytes | io.RawIOBase = b""
|
| 600 |
+
|
| 601 |
+
response = EmscriptenResponse(
|
| 602 |
+
status_code=status_code, headers=headers, body=b"", request=request
|
| 603 |
+
)
|
| 604 |
+
if streaming:
|
| 605 |
+
# get via inputstream
|
| 606 |
+
if response_js.body is not None:
|
| 607 |
+
# get a reader from the fetch response
|
| 608 |
+
body_stream_js = response_js.body.getReader()
|
| 609 |
+
body = _JSPIReadStream(
|
| 610 |
+
body_stream_js, timeout, request, response, js_abort_controller
|
| 611 |
+
)
|
| 612 |
+
else:
|
| 613 |
+
# get directly via arraybuffer
|
| 614 |
+
# n.b. this is another async JavaScript call.
|
| 615 |
+
body = _run_sync_with_timeout(
|
| 616 |
+
response_js.arrayBuffer(),
|
| 617 |
+
timeout,
|
| 618 |
+
js_abort_controller,
|
| 619 |
+
request=request,
|
| 620 |
+
response=response,
|
| 621 |
+
).to_py()
|
| 622 |
+
response.body = body
|
| 623 |
+
return response
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def _run_sync_with_timeout(
|
| 627 |
+
promise: Any,
|
| 628 |
+
timeout: float,
|
| 629 |
+
js_abort_controller: Any,
|
| 630 |
+
request: EmscriptenRequest | None,
|
| 631 |
+
response: EmscriptenResponse | None,
|
| 632 |
+
) -> Any:
|
| 633 |
+
"""
|
| 634 |
+
Await a JavaScript promise synchronously with a timeout which is implemented
|
| 635 |
+
via the AbortController
|
| 636 |
+
|
| 637 |
+
:param promise:
|
| 638 |
+
Javascript promise to await
|
| 639 |
+
|
| 640 |
+
:param timeout:
|
| 641 |
+
Timeout in seconds
|
| 642 |
+
|
| 643 |
+
:param js_abort_controller:
|
| 644 |
+
A JavaScript AbortController object, used on timeout
|
| 645 |
+
|
| 646 |
+
:param request:
|
| 647 |
+
The request being handled
|
| 648 |
+
|
| 649 |
+
:param response:
|
| 650 |
+
The response being handled (if it exists yet)
|
| 651 |
+
|
| 652 |
+
:raises _TimeoutError: If the request times out
|
| 653 |
+
:raises _RequestError: If the request raises a JavaScript exception
|
| 654 |
+
|
| 655 |
+
:return: The result of awaiting the promise.
|
| 656 |
+
"""
|
| 657 |
+
timer_id = None
|
| 658 |
+
if timeout > 0:
|
| 659 |
+
timer_id = js.setTimeout(
|
| 660 |
+
js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000)
|
| 661 |
+
)
|
| 662 |
+
try:
|
| 663 |
+
from pyodide.ffi import run_sync
|
| 664 |
+
|
| 665 |
+
# run_sync here uses WebAssembly JavaScript Promise Integration to
|
| 666 |
+
# suspend python until the JavaScript promise resolves.
|
| 667 |
+
return run_sync(promise)
|
| 668 |
+
except JsException as err:
|
| 669 |
+
if err.name == "AbortError":
|
| 670 |
+
raise _TimeoutError(
|
| 671 |
+
message="Request timed out", request=request, response=response
|
| 672 |
+
)
|
| 673 |
+
else:
|
| 674 |
+
raise _RequestError(message=err.message, request=request, response=response)
|
| 675 |
+
finally:
|
| 676 |
+
if timer_id is not None:
|
| 677 |
+
js.clearTimeout(timer_id)
|
| 678 |
+
|
| 679 |
+
|
| 680 |
+
def has_jspi() -> bool:
|
| 681 |
+
"""
|
| 682 |
+
Return true if jspi can be used.
|
| 683 |
+
|
| 684 |
+
This requires both browser support and also WebAssembly
|
| 685 |
+
to be in the correct state - i.e. that the javascript
|
| 686 |
+
call into python was async not sync.
|
| 687 |
+
|
| 688 |
+
:return: True if jspi can be used.
|
| 689 |
+
:rtype: bool
|
| 690 |
+
"""
|
| 691 |
+
try:
|
| 692 |
+
from pyodide.ffi import can_run_sync, run_sync # noqa: F401
|
| 693 |
+
|
| 694 |
+
return bool(can_run_sync())
|
| 695 |
+
except ImportError:
|
| 696 |
+
return False
|
| 697 |
+
|
| 698 |
+
|
| 699 |
+
def _is_node_js() -> bool:
|
| 700 |
+
"""
|
| 701 |
+
Check if we are in Node.js.
|
| 702 |
+
|
| 703 |
+
:return: True if we are in Node.js.
|
| 704 |
+
:rtype: bool
|
| 705 |
+
"""
|
| 706 |
+
return (
|
| 707 |
+
hasattr(js, "process")
|
| 708 |
+
and hasattr(js.process, "release")
|
| 709 |
+
# According to the Node.js documentation, the release name is always "node".
|
| 710 |
+
and js.process.release.name == "node"
|
| 711 |
+
)
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def streaming_ready() -> bool | None:
|
| 715 |
+
if _fetcher:
|
| 716 |
+
return _fetcher.streaming_ready
|
| 717 |
+
else:
|
| 718 |
+
return None # no fetcher, return None to signify that
|
| 719 |
+
|
| 720 |
+
|
| 721 |
+
async def wait_for_streaming_ready() -> bool:
|
| 722 |
+
if _fetcher:
|
| 723 |
+
await _fetcher.js_worker_ready_promise
|
| 724 |
+
return True
|
| 725 |
+
else:
|
| 726 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/request.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass, field
|
| 4 |
+
|
| 5 |
+
from ..._base_connection import _TYPE_BODY
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
@dataclass
|
| 9 |
+
class EmscriptenRequest:
|
| 10 |
+
method: str
|
| 11 |
+
url: str
|
| 12 |
+
params: dict[str, str] | None = None
|
| 13 |
+
body: _TYPE_BODY | None = None
|
| 14 |
+
headers: dict[str, str] = field(default_factory=dict)
|
| 15 |
+
timeout: float = 0
|
| 16 |
+
decode_content: bool = True
|
| 17 |
+
|
| 18 |
+
def set_header(self, name: str, value: str) -> None:
|
| 19 |
+
self.headers[name.capitalize()] = value
|
| 20 |
+
|
| 21 |
+
def set_body(self, body: _TYPE_BODY | None) -> None:
|
| 22 |
+
self.body = body
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/emscripten/response.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json as _json
|
| 4 |
+
import logging
|
| 5 |
+
import typing
|
| 6 |
+
from contextlib import contextmanager
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from http.client import HTTPException as HTTPException
|
| 9 |
+
from io import BytesIO, IOBase
|
| 10 |
+
|
| 11 |
+
from ...exceptions import InvalidHeader, TimeoutError
|
| 12 |
+
from ...response import BaseHTTPResponse
|
| 13 |
+
from ...util.retry import Retry
|
| 14 |
+
from .request import EmscriptenRequest
|
| 15 |
+
|
| 16 |
+
if typing.TYPE_CHECKING:
|
| 17 |
+
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
|
| 18 |
+
|
| 19 |
+
log = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class EmscriptenResponse:
|
| 24 |
+
status_code: int
|
| 25 |
+
headers: dict[str, str]
|
| 26 |
+
body: IOBase | bytes
|
| 27 |
+
request: EmscriptenRequest
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
|
| 31 |
+
def __init__(
|
| 32 |
+
self,
|
| 33 |
+
internal_response: EmscriptenResponse,
|
| 34 |
+
url: str | None = None,
|
| 35 |
+
connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
|
| 36 |
+
):
|
| 37 |
+
self._pool = None # set by pool class
|
| 38 |
+
self._body = None
|
| 39 |
+
self._response = internal_response
|
| 40 |
+
self._url = url
|
| 41 |
+
self._connection = connection
|
| 42 |
+
self._closed = False
|
| 43 |
+
super().__init__(
|
| 44 |
+
headers=internal_response.headers,
|
| 45 |
+
status=internal_response.status_code,
|
| 46 |
+
request_url=url,
|
| 47 |
+
version=0,
|
| 48 |
+
version_string="HTTP/?",
|
| 49 |
+
reason="",
|
| 50 |
+
decode_content=True,
|
| 51 |
+
)
|
| 52 |
+
self.length_remaining = self._init_length(self._response.request.method)
|
| 53 |
+
self.length_is_certain = False
|
| 54 |
+
|
| 55 |
+
@property
|
| 56 |
+
def url(self) -> str | None:
|
| 57 |
+
return self._url
|
| 58 |
+
|
| 59 |
+
@url.setter
|
| 60 |
+
def url(self, url: str | None) -> None:
|
| 61 |
+
self._url = url
|
| 62 |
+
|
| 63 |
+
@property
|
| 64 |
+
def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
|
| 65 |
+
return self._connection
|
| 66 |
+
|
| 67 |
+
@property
|
| 68 |
+
def retries(self) -> Retry | None:
|
| 69 |
+
return self._retries
|
| 70 |
+
|
| 71 |
+
@retries.setter
|
| 72 |
+
def retries(self, retries: Retry | None) -> None:
|
| 73 |
+
# Override the request_url if retries has a redirect location.
|
| 74 |
+
self._retries = retries
|
| 75 |
+
|
| 76 |
+
def stream(
|
| 77 |
+
self, amt: int | None = 2**16, decode_content: bool | None = None
|
| 78 |
+
) -> typing.Generator[bytes]:
|
| 79 |
+
"""
|
| 80 |
+
A generator wrapper for the read() method. A call will block until
|
| 81 |
+
``amt`` bytes have been read from the connection or until the
|
| 82 |
+
connection is closed.
|
| 83 |
+
|
| 84 |
+
:param amt:
|
| 85 |
+
How much of the content to read. The generator will return up to
|
| 86 |
+
much data per iteration, but may return less. This is particularly
|
| 87 |
+
likely when using compressed data. However, the empty string will
|
| 88 |
+
never be returned.
|
| 89 |
+
|
| 90 |
+
:param decode_content:
|
| 91 |
+
If True, will attempt to decode the body based on the
|
| 92 |
+
'content-encoding' header.
|
| 93 |
+
"""
|
| 94 |
+
while True:
|
| 95 |
+
data = self.read(amt=amt, decode_content=decode_content)
|
| 96 |
+
|
| 97 |
+
if data:
|
| 98 |
+
yield data
|
| 99 |
+
else:
|
| 100 |
+
break
|
| 101 |
+
|
| 102 |
+
def _init_length(self, request_method: str | None) -> int | None:
|
| 103 |
+
length: int | None
|
| 104 |
+
content_length: str | None = self.headers.get("content-length")
|
| 105 |
+
|
| 106 |
+
if content_length is not None:
|
| 107 |
+
try:
|
| 108 |
+
# RFC 7230 section 3.3.2 specifies multiple content lengths can
|
| 109 |
+
# be sent in a single Content-Length header
|
| 110 |
+
# (e.g. Content-Length: 42, 42). This line ensures the values
|
| 111 |
+
# are all valid ints and that as long as the `set` length is 1,
|
| 112 |
+
# all values are the same. Otherwise, the header is invalid.
|
| 113 |
+
lengths = {int(val) for val in content_length.split(",")}
|
| 114 |
+
if len(lengths) > 1:
|
| 115 |
+
raise InvalidHeader(
|
| 116 |
+
"Content-Length contained multiple "
|
| 117 |
+
"unmatching values (%s)" % content_length
|
| 118 |
+
)
|
| 119 |
+
length = lengths.pop()
|
| 120 |
+
except ValueError:
|
| 121 |
+
length = None
|
| 122 |
+
else:
|
| 123 |
+
if length < 0:
|
| 124 |
+
length = None
|
| 125 |
+
|
| 126 |
+
else: # if content_length is None
|
| 127 |
+
length = None
|
| 128 |
+
|
| 129 |
+
# Check for responses that shouldn't include a body
|
| 130 |
+
if (
|
| 131 |
+
self.status in (204, 304)
|
| 132 |
+
or 100 <= self.status < 200
|
| 133 |
+
or request_method == "HEAD"
|
| 134 |
+
):
|
| 135 |
+
length = 0
|
| 136 |
+
|
| 137 |
+
return length
|
| 138 |
+
|
| 139 |
+
def read(
|
| 140 |
+
self,
|
| 141 |
+
amt: int | None = None,
|
| 142 |
+
decode_content: bool | None = None, # ignored because browser decodes always
|
| 143 |
+
cache_content: bool = False,
|
| 144 |
+
) -> bytes:
|
| 145 |
+
if (
|
| 146 |
+
self._closed
|
| 147 |
+
or self._response is None
|
| 148 |
+
or (isinstance(self._response.body, IOBase) and self._response.body.closed)
|
| 149 |
+
):
|
| 150 |
+
return b""
|
| 151 |
+
|
| 152 |
+
with self._error_catcher():
|
| 153 |
+
# body has been preloaded as a string by XmlHttpRequest
|
| 154 |
+
if not isinstance(self._response.body, IOBase):
|
| 155 |
+
self.length_remaining = len(self._response.body)
|
| 156 |
+
self.length_is_certain = True
|
| 157 |
+
# wrap body in IOStream
|
| 158 |
+
self._response.body = BytesIO(self._response.body)
|
| 159 |
+
if amt is not None and amt >= 0:
|
| 160 |
+
# don't cache partial content
|
| 161 |
+
cache_content = False
|
| 162 |
+
data = self._response.body.read(amt)
|
| 163 |
+
else: # read all we can (and cache it)
|
| 164 |
+
data = self._response.body.read()
|
| 165 |
+
if cache_content:
|
| 166 |
+
self._body = data
|
| 167 |
+
if self.length_remaining is not None:
|
| 168 |
+
self.length_remaining = max(self.length_remaining - len(data), 0)
|
| 169 |
+
if len(data) == 0 or (
|
| 170 |
+
self.length_is_certain and self.length_remaining == 0
|
| 171 |
+
):
|
| 172 |
+
# definitely finished reading, close response stream
|
| 173 |
+
self._response.body.close()
|
| 174 |
+
return typing.cast(bytes, data)
|
| 175 |
+
|
| 176 |
+
def read_chunked(
|
| 177 |
+
self,
|
| 178 |
+
amt: int | None = None,
|
| 179 |
+
decode_content: bool | None = None,
|
| 180 |
+
) -> typing.Generator[bytes]:
|
| 181 |
+
# chunked is handled by browser
|
| 182 |
+
while True:
|
| 183 |
+
bytes = self.read(amt, decode_content)
|
| 184 |
+
if not bytes:
|
| 185 |
+
break
|
| 186 |
+
yield bytes
|
| 187 |
+
|
| 188 |
+
def release_conn(self) -> None:
|
| 189 |
+
if not self._pool or not self._connection:
|
| 190 |
+
return None
|
| 191 |
+
|
| 192 |
+
self._pool._put_conn(self._connection)
|
| 193 |
+
self._connection = None
|
| 194 |
+
|
| 195 |
+
def drain_conn(self) -> None:
|
| 196 |
+
self.close()
|
| 197 |
+
|
| 198 |
+
@property
|
| 199 |
+
def data(self) -> bytes:
|
| 200 |
+
if self._body:
|
| 201 |
+
return self._body
|
| 202 |
+
else:
|
| 203 |
+
return self.read(cache_content=True)
|
| 204 |
+
|
| 205 |
+
def json(self) -> typing.Any:
|
| 206 |
+
"""
|
| 207 |
+
Deserializes the body of the HTTP response as a Python object.
|
| 208 |
+
|
| 209 |
+
The body of the HTTP response must be encoded using UTF-8, as per
|
| 210 |
+
`RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
|
| 211 |
+
|
| 212 |
+
To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
|
| 213 |
+
your custom decoder instead.
|
| 214 |
+
|
| 215 |
+
If the body of the HTTP response is not decodable to UTF-8, a
|
| 216 |
+
`UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
|
| 217 |
+
valid JSON document, a `json.JSONDecodeError` will be raised.
|
| 218 |
+
|
| 219 |
+
Read more :ref:`here <json_content>`.
|
| 220 |
+
|
| 221 |
+
:returns: The body of the HTTP response as a Python object.
|
| 222 |
+
"""
|
| 223 |
+
data = self.data.decode("utf-8")
|
| 224 |
+
return _json.loads(data)
|
| 225 |
+
|
| 226 |
+
def close(self) -> None:
|
| 227 |
+
if not self._closed:
|
| 228 |
+
if isinstance(self._response.body, IOBase):
|
| 229 |
+
self._response.body.close()
|
| 230 |
+
if self._connection:
|
| 231 |
+
self._connection.close()
|
| 232 |
+
self._connection = None
|
| 233 |
+
self._closed = True
|
| 234 |
+
|
| 235 |
+
@contextmanager
|
| 236 |
+
def _error_catcher(self) -> typing.Generator[None]:
|
| 237 |
+
"""
|
| 238 |
+
Catch Emscripten specific exceptions thrown by fetch.py,
|
| 239 |
+
instead re-raising urllib3 variants, so that low-level exceptions
|
| 240 |
+
are not leaked in the high-level api.
|
| 241 |
+
|
| 242 |
+
On exit, release the connection back to the pool.
|
| 243 |
+
"""
|
| 244 |
+
from .fetch import _RequestError, _TimeoutError # avoid circular import
|
| 245 |
+
|
| 246 |
+
clean_exit = False
|
| 247 |
+
|
| 248 |
+
try:
|
| 249 |
+
yield
|
| 250 |
+
# If no exception is thrown, we should avoid cleaning up
|
| 251 |
+
# unnecessarily.
|
| 252 |
+
clean_exit = True
|
| 253 |
+
except _TimeoutError as e:
|
| 254 |
+
raise TimeoutError(str(e))
|
| 255 |
+
except _RequestError as e:
|
| 256 |
+
raise HTTPException(str(e))
|
| 257 |
+
finally:
|
| 258 |
+
# If we didn't terminate cleanly, we need to throw away our
|
| 259 |
+
# connection.
|
| 260 |
+
if not clean_exit:
|
| 261 |
+
# The response may not be closed but we're not going to use it
|
| 262 |
+
# anymore so close it now
|
| 263 |
+
if (
|
| 264 |
+
isinstance(self._response.body, IOBase)
|
| 265 |
+
and not self._response.body.closed
|
| 266 |
+
):
|
| 267 |
+
self._response.body.close()
|
| 268 |
+
# release the connection back to the pool
|
| 269 |
+
self.release_conn()
|
| 270 |
+
else:
|
| 271 |
+
# If we have read everything from the response stream,
|
| 272 |
+
# return the connection back to the pool.
|
| 273 |
+
if (
|
| 274 |
+
isinstance(self._response.body, IOBase)
|
| 275 |
+
and self._response.body.closed
|
| 276 |
+
):
|
| 277 |
+
self.release_conn()
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/pyopenssl.py
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Module for using pyOpenSSL as a TLS backend. This module was relevant before
|
| 3 |
+
the standard library ``ssl`` module supported SNI, but now that we've dropped
|
| 4 |
+
support for Python 2.7 all relevant Python versions support SNI so
|
| 5 |
+
**this module is no longer recommended**.
|
| 6 |
+
|
| 7 |
+
This needs the following packages installed:
|
| 8 |
+
|
| 9 |
+
* `pyOpenSSL`_ (tested with 16.0.0)
|
| 10 |
+
* `cryptography`_ (minimum 1.3.4, from pyopenssl)
|
| 11 |
+
* `idna`_ (minimum 2.0)
|
| 12 |
+
|
| 13 |
+
However, pyOpenSSL depends on cryptography, so while we use all three directly here we
|
| 14 |
+
end up having relatively few packages required.
|
| 15 |
+
|
| 16 |
+
You can install them with the following command:
|
| 17 |
+
|
| 18 |
+
.. code-block:: bash
|
| 19 |
+
|
| 20 |
+
$ python -m pip install pyopenssl cryptography idna
|
| 21 |
+
|
| 22 |
+
To activate certificate checking, call
|
| 23 |
+
:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code
|
| 24 |
+
before you begin making HTTP requests. This can be done in a ``sitecustomize``
|
| 25 |
+
module, or at any other time before your application begins using ``urllib3``,
|
| 26 |
+
like this:
|
| 27 |
+
|
| 28 |
+
.. code-block:: python
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
import urllib3.contrib.pyopenssl
|
| 32 |
+
urllib3.contrib.pyopenssl.inject_into_urllib3()
|
| 33 |
+
except ImportError:
|
| 34 |
+
pass
|
| 35 |
+
|
| 36 |
+
.. _pyopenssl: https://www.pyopenssl.org
|
| 37 |
+
.. _cryptography: https://cryptography.io
|
| 38 |
+
.. _idna: https://github.com/kjd/idna
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
import OpenSSL.SSL # type: ignore[import-not-found]
|
| 44 |
+
from cryptography import x509
|
| 45 |
+
|
| 46 |
+
try:
|
| 47 |
+
from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
|
| 48 |
+
except ImportError:
|
| 49 |
+
# UnsupportedExtension is gone in cryptography >= 2.1.0
|
| 50 |
+
class UnsupportedExtension(Exception): # type: ignore[no-redef]
|
| 51 |
+
pass
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
import logging
|
| 55 |
+
import ssl
|
| 56 |
+
import typing
|
| 57 |
+
from io import BytesIO
|
| 58 |
+
from socket import socket as socket_cls
|
| 59 |
+
from socket import timeout
|
| 60 |
+
|
| 61 |
+
from .. import util
|
| 62 |
+
|
| 63 |
+
if typing.TYPE_CHECKING:
|
| 64 |
+
from OpenSSL.crypto import X509 # type: ignore[import-not-found]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
|
| 68 |
+
|
| 69 |
+
# Map from urllib3 to PyOpenSSL compatible parameter-values.
|
| 70 |
+
_openssl_versions: dict[int, int] = {
|
| 71 |
+
util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
| 72 |
+
util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
| 73 |
+
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
|
| 77 |
+
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
|
| 78 |
+
|
| 79 |
+
if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"):
|
| 80 |
+
_openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
_stdlib_to_openssl_verify = {
|
| 84 |
+
ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
|
| 85 |
+
ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER,
|
| 86 |
+
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
|
| 87 |
+
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
|
| 88 |
+
}
|
| 89 |
+
_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
|
| 90 |
+
|
| 91 |
+
# The SSLvX values are the most likely to be missing in the future
|
| 92 |
+
# but we check them all just to be sure.
|
| 93 |
+
_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
|
| 94 |
+
OpenSSL.SSL, "OP_NO_SSLv3", 0
|
| 95 |
+
)
|
| 96 |
+
_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
|
| 97 |
+
_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
|
| 98 |
+
_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
|
| 99 |
+
_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
|
| 100 |
+
|
| 101 |
+
_openssl_to_ssl_minimum_version: dict[int, int] = {
|
| 102 |
+
ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
| 103 |
+
ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
|
| 104 |
+
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
|
| 105 |
+
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
|
| 106 |
+
ssl.TLSVersion.TLSv1_3: (
|
| 107 |
+
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
| 108 |
+
),
|
| 109 |
+
ssl.TLSVersion.MAXIMUM_SUPPORTED: (
|
| 110 |
+
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
| 111 |
+
),
|
| 112 |
+
}
|
| 113 |
+
_openssl_to_ssl_maximum_version: dict[int, int] = {
|
| 114 |
+
ssl.TLSVersion.MINIMUM_SUPPORTED: (
|
| 115 |
+
_OP_NO_SSLv2_OR_SSLv3
|
| 116 |
+
| _OP_NO_TLSv1
|
| 117 |
+
| _OP_NO_TLSv1_1
|
| 118 |
+
| _OP_NO_TLSv1_2
|
| 119 |
+
| _OP_NO_TLSv1_3
|
| 120 |
+
),
|
| 121 |
+
ssl.TLSVersion.TLSv1: (
|
| 122 |
+
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
|
| 123 |
+
),
|
| 124 |
+
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
|
| 125 |
+
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
|
| 126 |
+
ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
|
| 127 |
+
ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
# OpenSSL will only write 16K at a time
|
| 131 |
+
SSL_WRITE_BLOCKSIZE = 16384
|
| 132 |
+
|
| 133 |
+
orig_util_SSLContext = util.ssl_.SSLContext
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
log = logging.getLogger(__name__)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def inject_into_urllib3() -> None:
|
| 140 |
+
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
|
| 141 |
+
|
| 142 |
+
_validate_dependencies_met()
|
| 143 |
+
|
| 144 |
+
util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
| 145 |
+
util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
| 146 |
+
util.IS_PYOPENSSL = True
|
| 147 |
+
util.ssl_.IS_PYOPENSSL = True
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def extract_from_urllib3() -> None:
|
| 151 |
+
"Undo monkey-patching by :func:`inject_into_urllib3`."
|
| 152 |
+
|
| 153 |
+
util.SSLContext = orig_util_SSLContext
|
| 154 |
+
util.ssl_.SSLContext = orig_util_SSLContext
|
| 155 |
+
util.IS_PYOPENSSL = False
|
| 156 |
+
util.ssl_.IS_PYOPENSSL = False
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _validate_dependencies_met() -> None:
|
| 160 |
+
"""
|
| 161 |
+
Verifies that PyOpenSSL's package-level dependencies have been met.
|
| 162 |
+
Throws `ImportError` if they are not met.
|
| 163 |
+
"""
|
| 164 |
+
# Method added in `cryptography==1.1`; not available in older versions
|
| 165 |
+
from cryptography.x509.extensions import Extensions
|
| 166 |
+
|
| 167 |
+
if getattr(Extensions, "get_extension_for_class", None) is None:
|
| 168 |
+
raise ImportError(
|
| 169 |
+
"'cryptography' module missing required functionality. "
|
| 170 |
+
"Try upgrading to v1.3.4 or newer."
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
# pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
|
| 174 |
+
# attribute is only present on those versions.
|
| 175 |
+
from OpenSSL.crypto import X509
|
| 176 |
+
|
| 177 |
+
x509 = X509()
|
| 178 |
+
if getattr(x509, "_x509", None) is None:
|
| 179 |
+
raise ImportError(
|
| 180 |
+
"'pyOpenSSL' module missing required functionality. "
|
| 181 |
+
"Try upgrading to v0.14 or newer."
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _dnsname_to_stdlib(name: str) -> str | None:
|
| 186 |
+
"""
|
| 187 |
+
Converts a dNSName SubjectAlternativeName field to the form used by the
|
| 188 |
+
standard library on the given Python version.
|
| 189 |
+
|
| 190 |
+
Cryptography produces a dNSName as a unicode string that was idna-decoded
|
| 191 |
+
from ASCII bytes. We need to idna-encode that string to get it back, and
|
| 192 |
+
then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib
|
| 193 |
+
uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8).
|
| 194 |
+
|
| 195 |
+
If the name cannot be idna-encoded then we return None signalling that
|
| 196 |
+
the name given should be skipped.
|
| 197 |
+
"""
|
| 198 |
+
|
| 199 |
+
def idna_encode(name: str) -> bytes | None:
|
| 200 |
+
"""
|
| 201 |
+
Borrowed wholesale from the Python Cryptography Project. It turns out
|
| 202 |
+
that we can't just safely call `idna.encode`: it can explode for
|
| 203 |
+
wildcard names. This avoids that problem.
|
| 204 |
+
"""
|
| 205 |
+
import idna
|
| 206 |
+
|
| 207 |
+
try:
|
| 208 |
+
for prefix in ["*.", "."]:
|
| 209 |
+
if name.startswith(prefix):
|
| 210 |
+
name = name[len(prefix) :]
|
| 211 |
+
return prefix.encode("ascii") + idna.encode(name)
|
| 212 |
+
return idna.encode(name)
|
| 213 |
+
except idna.core.IDNAError:
|
| 214 |
+
return None
|
| 215 |
+
|
| 216 |
+
# Don't send IPv6 addresses through the IDNA encoder.
|
| 217 |
+
if ":" in name:
|
| 218 |
+
return name
|
| 219 |
+
|
| 220 |
+
encoded_name = idna_encode(name)
|
| 221 |
+
if encoded_name is None:
|
| 222 |
+
return None
|
| 223 |
+
return encoded_name.decode("utf-8")
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
|
| 227 |
+
"""
|
| 228 |
+
Given an PyOpenSSL certificate, provides all the subject alternative names.
|
| 229 |
+
"""
|
| 230 |
+
cert = peer_cert.to_cryptography()
|
| 231 |
+
|
| 232 |
+
# We want to find the SAN extension. Ask Cryptography to locate it (it's
|
| 233 |
+
# faster than looping in Python)
|
| 234 |
+
try:
|
| 235 |
+
ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value
|
| 236 |
+
except x509.ExtensionNotFound:
|
| 237 |
+
# No such extension, return the empty list.
|
| 238 |
+
return []
|
| 239 |
+
except (
|
| 240 |
+
x509.DuplicateExtension,
|
| 241 |
+
UnsupportedExtension,
|
| 242 |
+
x509.UnsupportedGeneralNameType,
|
| 243 |
+
UnicodeError,
|
| 244 |
+
) as e:
|
| 245 |
+
# A problem has been found with the quality of the certificate. Assume
|
| 246 |
+
# no SAN field is present.
|
| 247 |
+
log.warning(
|
| 248 |
+
"A problem was encountered with the certificate that prevented "
|
| 249 |
+
"urllib3 from finding the SubjectAlternativeName field. This can "
|
| 250 |
+
"affect certificate validation. The error was %s",
|
| 251 |
+
e,
|
| 252 |
+
)
|
| 253 |
+
return []
|
| 254 |
+
|
| 255 |
+
# We want to return dNSName and iPAddress fields. We need to cast the IPs
|
| 256 |
+
# back to strings because the match_hostname function wants them as
|
| 257 |
+
# strings.
|
| 258 |
+
# Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8
|
| 259 |
+
# decoded. This is pretty frustrating, but that's what the standard library
|
| 260 |
+
# does with certificates, and so we need to attempt to do the same.
|
| 261 |
+
# We also want to skip over names which cannot be idna encoded.
|
| 262 |
+
names = [
|
| 263 |
+
("DNS", name)
|
| 264 |
+
for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName))
|
| 265 |
+
if name is not None
|
| 266 |
+
]
|
| 267 |
+
names.extend(
|
| 268 |
+
("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress)
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
return names
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
class WrappedSocket:
|
| 275 |
+
"""API-compatibility wrapper for Python OpenSSL's Connection-class."""
|
| 276 |
+
|
| 277 |
+
def __init__(
|
| 278 |
+
self,
|
| 279 |
+
connection: OpenSSL.SSL.Connection,
|
| 280 |
+
socket: socket_cls,
|
| 281 |
+
suppress_ragged_eofs: bool = True,
|
| 282 |
+
) -> None:
|
| 283 |
+
self.connection = connection
|
| 284 |
+
self.socket = socket
|
| 285 |
+
self.suppress_ragged_eofs = suppress_ragged_eofs
|
| 286 |
+
self._io_refs = 0
|
| 287 |
+
self._closed = False
|
| 288 |
+
|
| 289 |
+
def fileno(self) -> int:
|
| 290 |
+
return self.socket.fileno()
|
| 291 |
+
|
| 292 |
+
# Copy-pasted from Python 3.5 source code
|
| 293 |
+
def _decref_socketios(self) -> None:
|
| 294 |
+
if self._io_refs > 0:
|
| 295 |
+
self._io_refs -= 1
|
| 296 |
+
if self._closed:
|
| 297 |
+
self.close()
|
| 298 |
+
|
| 299 |
+
def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
|
| 300 |
+
try:
|
| 301 |
+
data = self.connection.recv(*args, **kwargs)
|
| 302 |
+
except OpenSSL.SSL.SysCallError as e:
|
| 303 |
+
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
| 304 |
+
return b""
|
| 305 |
+
else:
|
| 306 |
+
raise OSError(e.args[0], str(e)) from e
|
| 307 |
+
except OpenSSL.SSL.ZeroReturnError:
|
| 308 |
+
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
| 309 |
+
return b""
|
| 310 |
+
else:
|
| 311 |
+
raise
|
| 312 |
+
except OpenSSL.SSL.WantReadError as e:
|
| 313 |
+
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
| 314 |
+
raise timeout("The read operation timed out") from e
|
| 315 |
+
else:
|
| 316 |
+
return self.recv(*args, **kwargs)
|
| 317 |
+
|
| 318 |
+
# TLS 1.3 post-handshake authentication
|
| 319 |
+
except OpenSSL.SSL.Error as e:
|
| 320 |
+
raise ssl.SSLError(f"read error: {e!r}") from e
|
| 321 |
+
else:
|
| 322 |
+
return data # type: ignore[no-any-return]
|
| 323 |
+
|
| 324 |
+
def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
|
| 325 |
+
try:
|
| 326 |
+
return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
|
| 327 |
+
except OpenSSL.SSL.SysCallError as e:
|
| 328 |
+
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
| 329 |
+
return 0
|
| 330 |
+
else:
|
| 331 |
+
raise OSError(e.args[0], str(e)) from e
|
| 332 |
+
except OpenSSL.SSL.ZeroReturnError:
|
| 333 |
+
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
| 334 |
+
return 0
|
| 335 |
+
else:
|
| 336 |
+
raise
|
| 337 |
+
except OpenSSL.SSL.WantReadError as e:
|
| 338 |
+
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
| 339 |
+
raise timeout("The read operation timed out") from e
|
| 340 |
+
else:
|
| 341 |
+
return self.recv_into(*args, **kwargs)
|
| 342 |
+
|
| 343 |
+
# TLS 1.3 post-handshake authentication
|
| 344 |
+
except OpenSSL.SSL.Error as e:
|
| 345 |
+
raise ssl.SSLError(f"read error: {e!r}") from e
|
| 346 |
+
|
| 347 |
+
def settimeout(self, timeout: float) -> None:
|
| 348 |
+
return self.socket.settimeout(timeout)
|
| 349 |
+
|
| 350 |
+
def _send_until_done(self, data: bytes) -> int:
|
| 351 |
+
while True:
|
| 352 |
+
try:
|
| 353 |
+
return self.connection.send(data) # type: ignore[no-any-return]
|
| 354 |
+
except OpenSSL.SSL.WantWriteError as e:
|
| 355 |
+
if not util.wait_for_write(self.socket, self.socket.gettimeout()):
|
| 356 |
+
raise timeout() from e
|
| 357 |
+
continue
|
| 358 |
+
except OpenSSL.SSL.SysCallError as e:
|
| 359 |
+
raise OSError(e.args[0], str(e)) from e
|
| 360 |
+
|
| 361 |
+
def sendall(self, data: bytes) -> None:
|
| 362 |
+
total_sent = 0
|
| 363 |
+
while total_sent < len(data):
|
| 364 |
+
sent = self._send_until_done(
|
| 365 |
+
data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]
|
| 366 |
+
)
|
| 367 |
+
total_sent += sent
|
| 368 |
+
|
| 369 |
+
def shutdown(self, how: int) -> None:
|
| 370 |
+
try:
|
| 371 |
+
self.connection.shutdown()
|
| 372 |
+
except OpenSSL.SSL.Error as e:
|
| 373 |
+
raise ssl.SSLError(f"shutdown error: {e!r}") from e
|
| 374 |
+
|
| 375 |
+
def close(self) -> None:
|
| 376 |
+
self._closed = True
|
| 377 |
+
if self._io_refs <= 0:
|
| 378 |
+
self._real_close()
|
| 379 |
+
|
| 380 |
+
def _real_close(self) -> None:
|
| 381 |
+
try:
|
| 382 |
+
return self.connection.close() # type: ignore[no-any-return]
|
| 383 |
+
except OpenSSL.SSL.Error:
|
| 384 |
+
return
|
| 385 |
+
|
| 386 |
+
def getpeercert(
|
| 387 |
+
self, binary_form: bool = False
|
| 388 |
+
) -> dict[str, list[typing.Any]] | None:
|
| 389 |
+
x509 = self.connection.get_peer_certificate()
|
| 390 |
+
|
| 391 |
+
if not x509:
|
| 392 |
+
return x509 # type: ignore[no-any-return]
|
| 393 |
+
|
| 394 |
+
if binary_form:
|
| 395 |
+
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
|
| 396 |
+
|
| 397 |
+
return {
|
| 398 |
+
"subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
|
| 399 |
+
"subjectAltName": get_subj_alt_name(x509),
|
| 400 |
+
}
|
| 401 |
+
|
| 402 |
+
def version(self) -> str:
|
| 403 |
+
return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
|
| 404 |
+
|
| 405 |
+
def selected_alpn_protocol(self) -> str | None:
|
| 406 |
+
alpn_proto = self.connection.get_alpn_proto_negotiated()
|
| 407 |
+
return alpn_proto.decode() if alpn_proto else None
|
| 408 |
+
|
| 409 |
+
|
| 410 |
+
WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
class PyOpenSSLContext:
|
| 414 |
+
"""
|
| 415 |
+
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
|
| 416 |
+
for translating the interface of the standard library ``SSLContext`` object
|
| 417 |
+
to calls into PyOpenSSL.
|
| 418 |
+
"""
|
| 419 |
+
|
| 420 |
+
def __init__(self, protocol: int) -> None:
|
| 421 |
+
self.protocol = _openssl_versions[protocol]
|
| 422 |
+
self._ctx = OpenSSL.SSL.Context(self.protocol)
|
| 423 |
+
self._options = 0
|
| 424 |
+
self.check_hostname = False
|
| 425 |
+
self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
|
| 426 |
+
self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
|
| 427 |
+
self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST
|
| 428 |
+
|
| 429 |
+
@property
|
| 430 |
+
def options(self) -> int:
|
| 431 |
+
return self._options
|
| 432 |
+
|
| 433 |
+
@options.setter
|
| 434 |
+
def options(self, value: int) -> None:
|
| 435 |
+
self._options = value
|
| 436 |
+
self._set_ctx_options()
|
| 437 |
+
|
| 438 |
+
@property
|
| 439 |
+
def verify_flags(self) -> int:
|
| 440 |
+
return self._verify_flags
|
| 441 |
+
|
| 442 |
+
@verify_flags.setter
|
| 443 |
+
def verify_flags(self, value: int) -> None:
|
| 444 |
+
self._verify_flags = value
|
| 445 |
+
self._ctx.get_cert_store().set_flags(self._verify_flags)
|
| 446 |
+
|
| 447 |
+
@property
|
| 448 |
+
def verify_mode(self) -> int:
|
| 449 |
+
return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
|
| 450 |
+
|
| 451 |
+
@verify_mode.setter
|
| 452 |
+
def verify_mode(self, value: ssl.VerifyMode) -> None:
|
| 453 |
+
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
|
| 454 |
+
|
| 455 |
+
def set_default_verify_paths(self) -> None:
|
| 456 |
+
self._ctx.set_default_verify_paths()
|
| 457 |
+
|
| 458 |
+
def set_ciphers(self, ciphers: bytes | str) -> None:
|
| 459 |
+
if isinstance(ciphers, str):
|
| 460 |
+
ciphers = ciphers.encode("utf-8")
|
| 461 |
+
self._ctx.set_cipher_list(ciphers)
|
| 462 |
+
|
| 463 |
+
def load_verify_locations(
|
| 464 |
+
self,
|
| 465 |
+
cafile: str | None = None,
|
| 466 |
+
capath: str | None = None,
|
| 467 |
+
cadata: bytes | None = None,
|
| 468 |
+
) -> None:
|
| 469 |
+
if cafile is not None:
|
| 470 |
+
cafile = cafile.encode("utf-8") # type: ignore[assignment]
|
| 471 |
+
if capath is not None:
|
| 472 |
+
capath = capath.encode("utf-8") # type: ignore[assignment]
|
| 473 |
+
try:
|
| 474 |
+
self._ctx.load_verify_locations(cafile, capath)
|
| 475 |
+
if cadata is not None:
|
| 476 |
+
self._ctx.load_verify_locations(BytesIO(cadata))
|
| 477 |
+
except OpenSSL.SSL.Error as e:
|
| 478 |
+
raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
|
| 479 |
+
|
| 480 |
+
def load_cert_chain(
|
| 481 |
+
self,
|
| 482 |
+
certfile: str,
|
| 483 |
+
keyfile: str | None = None,
|
| 484 |
+
password: str | None = None,
|
| 485 |
+
) -> None:
|
| 486 |
+
try:
|
| 487 |
+
self._ctx.use_certificate_chain_file(certfile)
|
| 488 |
+
if password is not None:
|
| 489 |
+
if not isinstance(password, bytes):
|
| 490 |
+
password = password.encode("utf-8") # type: ignore[assignment]
|
| 491 |
+
self._ctx.set_passwd_cb(lambda *_: password)
|
| 492 |
+
self._ctx.use_privatekey_file(keyfile or certfile)
|
| 493 |
+
except OpenSSL.SSL.Error as e:
|
| 494 |
+
raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
|
| 495 |
+
|
| 496 |
+
def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
|
| 497 |
+
protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
|
| 498 |
+
return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
|
| 499 |
+
|
| 500 |
+
def wrap_socket(
|
| 501 |
+
self,
|
| 502 |
+
sock: socket_cls,
|
| 503 |
+
server_side: bool = False,
|
| 504 |
+
do_handshake_on_connect: bool = True,
|
| 505 |
+
suppress_ragged_eofs: bool = True,
|
| 506 |
+
server_hostname: bytes | str | None = None,
|
| 507 |
+
) -> WrappedSocket:
|
| 508 |
+
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
|
| 509 |
+
|
| 510 |
+
# If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
|
| 511 |
+
if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
|
| 512 |
+
if isinstance(server_hostname, str):
|
| 513 |
+
server_hostname = server_hostname.encode("utf-8")
|
| 514 |
+
cnx.set_tlsext_host_name(server_hostname)
|
| 515 |
+
|
| 516 |
+
cnx.set_connect_state()
|
| 517 |
+
|
| 518 |
+
while True:
|
| 519 |
+
try:
|
| 520 |
+
cnx.do_handshake()
|
| 521 |
+
except OpenSSL.SSL.WantReadError as e:
|
| 522 |
+
if not util.wait_for_read(sock, sock.gettimeout()):
|
| 523 |
+
raise timeout("select timed out") from e
|
| 524 |
+
continue
|
| 525 |
+
except OpenSSL.SSL.Error as e:
|
| 526 |
+
raise ssl.SSLError(f"bad handshake: {e!r}") from e
|
| 527 |
+
break
|
| 528 |
+
|
| 529 |
+
return WrappedSocket(cnx, sock)
|
| 530 |
+
|
| 531 |
+
def _set_ctx_options(self) -> None:
|
| 532 |
+
self._ctx.set_options(
|
| 533 |
+
self._options
|
| 534 |
+
| _openssl_to_ssl_minimum_version[self._minimum_version]
|
| 535 |
+
| _openssl_to_ssl_maximum_version[self._maximum_version]
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
@property
|
| 539 |
+
def minimum_version(self) -> int:
|
| 540 |
+
return self._minimum_version
|
| 541 |
+
|
| 542 |
+
@minimum_version.setter
|
| 543 |
+
def minimum_version(self, minimum_version: int) -> None:
|
| 544 |
+
self._minimum_version = minimum_version
|
| 545 |
+
self._set_ctx_options()
|
| 546 |
+
|
| 547 |
+
@property
|
| 548 |
+
def maximum_version(self) -> int:
|
| 549 |
+
return self._maximum_version
|
| 550 |
+
|
| 551 |
+
@maximum_version.setter
|
| 552 |
+
def maximum_version(self, maximum_version: int) -> None:
|
| 553 |
+
self._maximum_version = maximum_version
|
| 554 |
+
self._set_ctx_options()
|
| 555 |
+
|
| 556 |
+
|
| 557 |
+
def _verify_callback(
|
| 558 |
+
cnx: OpenSSL.SSL.Connection,
|
| 559 |
+
x509: X509,
|
| 560 |
+
err_no: int,
|
| 561 |
+
err_depth: int,
|
| 562 |
+
return_code: int,
|
| 563 |
+
) -> bool:
|
| 564 |
+
return err_no == 0
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/contrib/socks.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
This module contains provisional support for SOCKS proxies from within
|
| 3 |
+
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
|
| 4 |
+
SOCKS5. To enable its functionality, either install PySocks or install this
|
| 5 |
+
module with the ``socks`` extra.
|
| 6 |
+
|
| 7 |
+
The SOCKS implementation supports the full range of urllib3 features. It also
|
| 8 |
+
supports the following SOCKS features:
|
| 9 |
+
|
| 10 |
+
- SOCKS4A (``proxy_url='socks4a://...``)
|
| 11 |
+
- SOCKS4 (``proxy_url='socks4://...``)
|
| 12 |
+
- SOCKS5 with remote DNS (``proxy_url='socks5h://...``)
|
| 13 |
+
- SOCKS5 with local DNS (``proxy_url='socks5://...``)
|
| 14 |
+
- Usernames and passwords for the SOCKS proxy
|
| 15 |
+
|
| 16 |
+
.. note::
|
| 17 |
+
It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in
|
| 18 |
+
your ``proxy_url`` to ensure that DNS resolution is done from the remote
|
| 19 |
+
server instead of client-side when connecting to a domain name.
|
| 20 |
+
|
| 21 |
+
SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5
|
| 22 |
+
supports IPv4, IPv6, and domain names.
|
| 23 |
+
|
| 24 |
+
When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url``
|
| 25 |
+
will be sent as the ``userid`` section of the SOCKS request:
|
| 26 |
+
|
| 27 |
+
.. code-block:: python
|
| 28 |
+
|
| 29 |
+
proxy_url="socks4a://<userid>@proxy-host"
|
| 30 |
+
|
| 31 |
+
When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion
|
| 32 |
+
of the ``proxy_url`` will be sent as the username/password to authenticate
|
| 33 |
+
with the proxy:
|
| 34 |
+
|
| 35 |
+
.. code-block:: python
|
| 36 |
+
|
| 37 |
+
proxy_url="socks5h://<username>:<password>@proxy-host"
|
| 38 |
+
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
import socks # type: ignore[import-untyped]
|
| 45 |
+
except ImportError:
|
| 46 |
+
import warnings
|
| 47 |
+
|
| 48 |
+
from ..exceptions import DependencyWarning
|
| 49 |
+
|
| 50 |
+
warnings.warn(
|
| 51 |
+
(
|
| 52 |
+
"SOCKS support in urllib3 requires the installation of optional "
|
| 53 |
+
"dependencies: specifically, PySocks. For more information, see "
|
| 54 |
+
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
|
| 55 |
+
),
|
| 56 |
+
DependencyWarning,
|
| 57 |
+
)
|
| 58 |
+
raise
|
| 59 |
+
|
| 60 |
+
import typing
|
| 61 |
+
from socket import timeout as SocketTimeout
|
| 62 |
+
|
| 63 |
+
from ..connection import HTTPConnection, HTTPSConnection
|
| 64 |
+
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
| 65 |
+
from ..exceptions import ConnectTimeoutError, NewConnectionError
|
| 66 |
+
from ..poolmanager import PoolManager
|
| 67 |
+
from ..util.url import parse_url
|
| 68 |
+
|
| 69 |
+
try:
|
| 70 |
+
import ssl
|
| 71 |
+
except ImportError:
|
| 72 |
+
ssl = None # type: ignore[assignment]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
|
| 76 |
+
socks_version: int
|
| 77 |
+
proxy_host: str | None
|
| 78 |
+
proxy_port: str | None
|
| 79 |
+
username: str | None
|
| 80 |
+
password: str | None
|
| 81 |
+
rdns: bool
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class SOCKSConnection(HTTPConnection):
|
| 85 |
+
"""
|
| 86 |
+
A plain-text HTTP connection that connects via a SOCKS proxy.
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
def __init__(
|
| 90 |
+
self,
|
| 91 |
+
_socks_options: _TYPE_SOCKS_OPTIONS,
|
| 92 |
+
*args: typing.Any,
|
| 93 |
+
**kwargs: typing.Any,
|
| 94 |
+
) -> None:
|
| 95 |
+
self._socks_options = _socks_options
|
| 96 |
+
super().__init__(*args, **kwargs)
|
| 97 |
+
|
| 98 |
+
def _new_conn(self) -> socks.socksocket:
|
| 99 |
+
"""
|
| 100 |
+
Establish a new connection via the SOCKS proxy.
|
| 101 |
+
"""
|
| 102 |
+
extra_kw: dict[str, typing.Any] = {}
|
| 103 |
+
if self.source_address:
|
| 104 |
+
extra_kw["source_address"] = self.source_address
|
| 105 |
+
|
| 106 |
+
if self.socket_options:
|
| 107 |
+
extra_kw["socket_options"] = self.socket_options
|
| 108 |
+
|
| 109 |
+
try:
|
| 110 |
+
conn = socks.create_connection(
|
| 111 |
+
(self.host, self.port),
|
| 112 |
+
proxy_type=self._socks_options["socks_version"],
|
| 113 |
+
proxy_addr=self._socks_options["proxy_host"],
|
| 114 |
+
proxy_port=self._socks_options["proxy_port"],
|
| 115 |
+
proxy_username=self._socks_options["username"],
|
| 116 |
+
proxy_password=self._socks_options["password"],
|
| 117 |
+
proxy_rdns=self._socks_options["rdns"],
|
| 118 |
+
timeout=self.timeout,
|
| 119 |
+
**extra_kw,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
except SocketTimeout as e:
|
| 123 |
+
raise ConnectTimeoutError(
|
| 124 |
+
self,
|
| 125 |
+
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
| 126 |
+
) from e
|
| 127 |
+
|
| 128 |
+
except socks.ProxyError as e:
|
| 129 |
+
# This is fragile as hell, but it seems to be the only way to raise
|
| 130 |
+
# useful errors here.
|
| 131 |
+
if e.socket_err:
|
| 132 |
+
error = e.socket_err
|
| 133 |
+
if isinstance(error, SocketTimeout):
|
| 134 |
+
raise ConnectTimeoutError(
|
| 135 |
+
self,
|
| 136 |
+
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
| 137 |
+
) from e
|
| 138 |
+
else:
|
| 139 |
+
# Adding `from e` messes with coverage somehow, so it's omitted.
|
| 140 |
+
# See #2386.
|
| 141 |
+
raise NewConnectionError(
|
| 142 |
+
self, f"Failed to establish a new connection: {error}"
|
| 143 |
+
)
|
| 144 |
+
else:
|
| 145 |
+
raise NewConnectionError(
|
| 146 |
+
self, f"Failed to establish a new connection: {e}"
|
| 147 |
+
) from e
|
| 148 |
+
|
| 149 |
+
except OSError as e: # Defensive: PySocks should catch all these.
|
| 150 |
+
raise NewConnectionError(
|
| 151 |
+
self, f"Failed to establish a new connection: {e}"
|
| 152 |
+
) from e
|
| 153 |
+
|
| 154 |
+
return conn
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
# We don't need to duplicate the Verified/Unverified distinction from
|
| 158 |
+
# urllib3/connection.py here because the HTTPSConnection will already have been
|
| 159 |
+
# correctly set to either the Verified or Unverified form by that module. This
|
| 160 |
+
# means the SOCKSHTTPSConnection will automatically be the correct type.
|
| 161 |
+
class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection):
|
| 162 |
+
pass
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class SOCKSHTTPConnectionPool(HTTPConnectionPool):
|
| 166 |
+
ConnectionCls = SOCKSConnection
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
class SOCKSHTTPSConnectionPool(HTTPSConnectionPool):
|
| 170 |
+
ConnectionCls = SOCKSHTTPSConnection
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
class SOCKSProxyManager(PoolManager):
|
| 174 |
+
"""
|
| 175 |
+
A version of the urllib3 ProxyManager that routes connections via the
|
| 176 |
+
defined SOCKS proxy.
|
| 177 |
+
"""
|
| 178 |
+
|
| 179 |
+
pool_classes_by_scheme = {
|
| 180 |
+
"http": SOCKSHTTPConnectionPool,
|
| 181 |
+
"https": SOCKSHTTPSConnectionPool,
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
def __init__(
|
| 185 |
+
self,
|
| 186 |
+
proxy_url: str,
|
| 187 |
+
username: str | None = None,
|
| 188 |
+
password: str | None = None,
|
| 189 |
+
num_pools: int = 10,
|
| 190 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 191 |
+
**connection_pool_kw: typing.Any,
|
| 192 |
+
):
|
| 193 |
+
parsed = parse_url(proxy_url)
|
| 194 |
+
|
| 195 |
+
if username is None and password is None and parsed.auth is not None:
|
| 196 |
+
split = parsed.auth.split(":")
|
| 197 |
+
if len(split) == 2:
|
| 198 |
+
username, password = split
|
| 199 |
+
if parsed.scheme == "socks5":
|
| 200 |
+
socks_version = socks.PROXY_TYPE_SOCKS5
|
| 201 |
+
rdns = False
|
| 202 |
+
elif parsed.scheme == "socks5h":
|
| 203 |
+
socks_version = socks.PROXY_TYPE_SOCKS5
|
| 204 |
+
rdns = True
|
| 205 |
+
elif parsed.scheme == "socks4":
|
| 206 |
+
socks_version = socks.PROXY_TYPE_SOCKS4
|
| 207 |
+
rdns = False
|
| 208 |
+
elif parsed.scheme == "socks4a":
|
| 209 |
+
socks_version = socks.PROXY_TYPE_SOCKS4
|
| 210 |
+
rdns = True
|
| 211 |
+
else:
|
| 212 |
+
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
|
| 213 |
+
|
| 214 |
+
self.proxy_url = proxy_url
|
| 215 |
+
|
| 216 |
+
socks_options = {
|
| 217 |
+
"socks_version": socks_version,
|
| 218 |
+
"proxy_host": parsed.host,
|
| 219 |
+
"proxy_port": parsed.port,
|
| 220 |
+
"username": username,
|
| 221 |
+
"password": password,
|
| 222 |
+
"rdns": rdns,
|
| 223 |
+
}
|
| 224 |
+
connection_pool_kw["_socks_options"] = socks_options
|
| 225 |
+
|
| 226 |
+
super().__init__(num_pools, headers, **connection_pool_kw)
|
| 227 |
+
|
| 228 |
+
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/exceptions.py
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import socket
|
| 4 |
+
import typing
|
| 5 |
+
import warnings
|
| 6 |
+
from email.errors import MessageDefect
|
| 7 |
+
from http.client import IncompleteRead as httplib_IncompleteRead
|
| 8 |
+
|
| 9 |
+
if typing.TYPE_CHECKING:
|
| 10 |
+
from .connection import HTTPConnection
|
| 11 |
+
from .connectionpool import ConnectionPool
|
| 12 |
+
from .response import HTTPResponse
|
| 13 |
+
from .util.retry import Retry
|
| 14 |
+
|
| 15 |
+
# Base Exceptions
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class HTTPError(Exception):
|
| 19 |
+
"""Base exception used by this module."""
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class HTTPWarning(Warning):
|
| 23 |
+
"""Base warning used by this module."""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class PoolError(HTTPError):
|
| 30 |
+
"""Base exception for errors caused within a pool."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, pool: ConnectionPool, message: str) -> None:
|
| 33 |
+
self.pool = pool
|
| 34 |
+
self._message = message
|
| 35 |
+
super().__init__(f"{pool}: {message}")
|
| 36 |
+
|
| 37 |
+
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
| 38 |
+
# For pickling purposes.
|
| 39 |
+
return self.__class__, (None, self._message)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class RequestError(PoolError):
|
| 43 |
+
"""Base exception for PoolErrors that have associated URLs."""
|
| 44 |
+
|
| 45 |
+
def __init__(self, pool: ConnectionPool, url: str | None, message: str) -> None:
|
| 46 |
+
self.url = url
|
| 47 |
+
super().__init__(pool, message)
|
| 48 |
+
|
| 49 |
+
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
| 50 |
+
# For pickling purposes.
|
| 51 |
+
return self.__class__, (None, self.url, self._message)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class SSLError(HTTPError):
|
| 55 |
+
"""Raised when SSL certificate fails in an HTTPS connection."""
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class ProxyError(HTTPError):
|
| 59 |
+
"""Raised when the connection to a proxy fails."""
|
| 60 |
+
|
| 61 |
+
# The original error is also available as __cause__.
|
| 62 |
+
original_error: Exception
|
| 63 |
+
|
| 64 |
+
def __init__(self, message: str, error: Exception) -> None:
|
| 65 |
+
super().__init__(message, error)
|
| 66 |
+
self.original_error = error
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class DecodeError(HTTPError):
|
| 70 |
+
"""Raised when automatic decoding based on Content-Type fails."""
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class ProtocolError(HTTPError):
|
| 74 |
+
"""Raised when something unexpected happens mid-request/response."""
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
#: Renamed to ProtocolError but aliased for backwards compatibility.
|
| 78 |
+
ConnectionError = ProtocolError
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# Leaf Exceptions
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class MaxRetryError(RequestError):
|
| 85 |
+
"""Raised when the maximum number of retries is exceeded.
|
| 86 |
+
|
| 87 |
+
:param pool: The connection pool
|
| 88 |
+
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
|
| 89 |
+
:param str url: The requested Url
|
| 90 |
+
:param reason: The underlying error
|
| 91 |
+
:type reason: :class:`Exception`
|
| 92 |
+
|
| 93 |
+
"""
|
| 94 |
+
|
| 95 |
+
def __init__(
|
| 96 |
+
self, pool: ConnectionPool, url: str | None, reason: Exception | None = None
|
| 97 |
+
) -> None:
|
| 98 |
+
self.reason = reason
|
| 99 |
+
|
| 100 |
+
message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
|
| 101 |
+
|
| 102 |
+
super().__init__(pool, url, message)
|
| 103 |
+
|
| 104 |
+
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
| 105 |
+
# For pickling purposes.
|
| 106 |
+
return self.__class__, (None, self.url, self.reason)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class HostChangedError(RequestError):
|
| 110 |
+
"""Raised when an existing pool gets a request for a foreign host."""
|
| 111 |
+
|
| 112 |
+
def __init__(
|
| 113 |
+
self, pool: ConnectionPool, url: str, retries: Retry | int = 3
|
| 114 |
+
) -> None:
|
| 115 |
+
message = f"Tried to open a foreign host with url: {url}"
|
| 116 |
+
super().__init__(pool, url, message)
|
| 117 |
+
self.retries = retries
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class TimeoutStateError(HTTPError):
|
| 121 |
+
"""Raised when passing an invalid state to a timeout"""
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
class TimeoutError(HTTPError):
|
| 125 |
+
"""Raised when a socket timeout error occurs.
|
| 126 |
+
|
| 127 |
+
Catching this error will catch both :exc:`ReadTimeoutErrors
|
| 128 |
+
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
|
| 129 |
+
"""
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class ReadTimeoutError(TimeoutError, RequestError):
|
| 133 |
+
"""Raised when a socket timeout occurs while receiving data from a server"""
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
# This timeout error does not have a URL attached and needs to inherit from the
|
| 137 |
+
# base HTTPError
|
| 138 |
+
class ConnectTimeoutError(TimeoutError):
|
| 139 |
+
"""Raised when a socket timeout occurs while connecting to a server"""
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
class NewConnectionError(ConnectTimeoutError, HTTPError):
|
| 143 |
+
"""Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
|
| 144 |
+
|
| 145 |
+
def __init__(self, conn: HTTPConnection, message: str) -> None:
|
| 146 |
+
self.conn = conn
|
| 147 |
+
self._message = message
|
| 148 |
+
super().__init__(f"{conn}: {message}")
|
| 149 |
+
|
| 150 |
+
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
| 151 |
+
# For pickling purposes.
|
| 152 |
+
return self.__class__, (None, self._message)
|
| 153 |
+
|
| 154 |
+
@property
|
| 155 |
+
def pool(self) -> HTTPConnection:
|
| 156 |
+
warnings.warn(
|
| 157 |
+
"The 'pool' property is deprecated and will be removed "
|
| 158 |
+
"in urllib3 v2.1.0. Use 'conn' instead.",
|
| 159 |
+
DeprecationWarning,
|
| 160 |
+
stacklevel=2,
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
return self.conn
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
class NameResolutionError(NewConnectionError):
|
| 167 |
+
"""Raised when host name resolution fails."""
|
| 168 |
+
|
| 169 |
+
def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
|
| 170 |
+
message = f"Failed to resolve '{host}' ({reason})"
|
| 171 |
+
self._host = host
|
| 172 |
+
self._reason = reason
|
| 173 |
+
super().__init__(conn, message)
|
| 174 |
+
|
| 175 |
+
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
| 176 |
+
# For pickling purposes.
|
| 177 |
+
return self.__class__, (self._host, None, self._reason)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class EmptyPoolError(PoolError):
|
| 181 |
+
"""Raised when a pool runs out of connections and no more are allowed."""
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
class FullPoolError(PoolError):
|
| 185 |
+
"""Raised when we try to add a connection to a full pool in blocking mode."""
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
class ClosedPoolError(PoolError):
|
| 189 |
+
"""Raised when a request enters a pool after the pool has been closed."""
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
class LocationValueError(ValueError, HTTPError):
|
| 193 |
+
"""Raised when there is something wrong with a given URL input."""
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class LocationParseError(LocationValueError):
|
| 197 |
+
"""Raised when get_host or similar fails to parse the URL input."""
|
| 198 |
+
|
| 199 |
+
def __init__(self, location: str) -> None:
|
| 200 |
+
message = f"Failed to parse: {location}"
|
| 201 |
+
super().__init__(message)
|
| 202 |
+
|
| 203 |
+
self.location = location
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
class URLSchemeUnknown(LocationValueError):
|
| 207 |
+
"""Raised when a URL input has an unsupported scheme."""
|
| 208 |
+
|
| 209 |
+
def __init__(self, scheme: str):
|
| 210 |
+
message = f"Not supported URL scheme {scheme}"
|
| 211 |
+
super().__init__(message)
|
| 212 |
+
|
| 213 |
+
self.scheme = scheme
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
class ResponseError(HTTPError):
|
| 217 |
+
"""Used as a container for an error reason supplied in a MaxRetryError."""
|
| 218 |
+
|
| 219 |
+
GENERIC_ERROR = "too many error responses"
|
| 220 |
+
SPECIFIC_ERROR = "too many {status_code} error responses"
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
class SecurityWarning(HTTPWarning):
|
| 224 |
+
"""Warned when performing security reducing actions"""
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class InsecureRequestWarning(SecurityWarning):
|
| 228 |
+
"""Warned when making an unverified HTTPS request."""
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
class NotOpenSSLWarning(SecurityWarning):
|
| 232 |
+
"""Warned when using unsupported SSL library"""
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
class SystemTimeWarning(SecurityWarning):
|
| 236 |
+
"""Warned when system time is suspected to be wrong"""
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
class InsecurePlatformWarning(SecurityWarning):
|
| 240 |
+
"""Warned when certain TLS/SSL configuration is not available on a platform."""
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
class DependencyWarning(HTTPWarning):
|
| 244 |
+
"""
|
| 245 |
+
Warned when an attempt is made to import a module with missing optional
|
| 246 |
+
dependencies.
|
| 247 |
+
"""
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
class ResponseNotChunked(ProtocolError, ValueError):
|
| 251 |
+
"""Response needs to be chunked in order to read it as chunks."""
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
class BodyNotHttplibCompatible(HTTPError):
|
| 255 |
+
"""
|
| 256 |
+
Body should be :class:`http.client.HTTPResponse` like
|
| 257 |
+
(have an fp attribute which returns raw chunks) for read_chunked().
|
| 258 |
+
"""
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
class IncompleteRead(HTTPError, httplib_IncompleteRead):
|
| 262 |
+
"""
|
| 263 |
+
Response length doesn't match expected Content-Length
|
| 264 |
+
|
| 265 |
+
Subclass of :class:`http.client.IncompleteRead` to allow int value
|
| 266 |
+
for ``partial`` to avoid creating large objects on streamed reads.
|
| 267 |
+
"""
|
| 268 |
+
|
| 269 |
+
partial: int # type: ignore[assignment]
|
| 270 |
+
expected: int
|
| 271 |
+
|
| 272 |
+
def __init__(self, partial: int, expected: int) -> None:
|
| 273 |
+
self.partial = partial
|
| 274 |
+
self.expected = expected
|
| 275 |
+
|
| 276 |
+
def __repr__(self) -> str:
|
| 277 |
+
return "IncompleteRead(%i bytes read, %i more expected)" % (
|
| 278 |
+
self.partial,
|
| 279 |
+
self.expected,
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
|
| 284 |
+
"""Invalid chunk length in a chunked response."""
|
| 285 |
+
|
| 286 |
+
def __init__(self, response: HTTPResponse, length: bytes) -> None:
|
| 287 |
+
self.partial: int = response.tell() # type: ignore[assignment]
|
| 288 |
+
self.expected: int | None = response.length_remaining
|
| 289 |
+
self.response = response
|
| 290 |
+
self.length = length
|
| 291 |
+
|
| 292 |
+
def __repr__(self) -> str:
|
| 293 |
+
return "InvalidChunkLength(got length %r, %i bytes read)" % (
|
| 294 |
+
self.length,
|
| 295 |
+
self.partial,
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
class InvalidHeader(HTTPError):
|
| 300 |
+
"""The header provided was somehow invalid."""
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
|
| 304 |
+
"""ProxyManager does not support the supplied scheme"""
|
| 305 |
+
|
| 306 |
+
# TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
|
| 307 |
+
|
| 308 |
+
def __init__(self, scheme: str | None) -> None:
|
| 309 |
+
# 'localhost' is here because our URL parser parses
|
| 310 |
+
# localhost:8080 -> scheme=localhost, remove if we fix this.
|
| 311 |
+
if scheme == "localhost":
|
| 312 |
+
scheme = None
|
| 313 |
+
if scheme is None:
|
| 314 |
+
message = "Proxy URL had no scheme, should start with http:// or https://"
|
| 315 |
+
else:
|
| 316 |
+
message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
|
| 317 |
+
super().__init__(message)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
class ProxySchemeUnsupported(ValueError):
|
| 321 |
+
"""Fetching HTTPS resources through HTTPS proxies is unsupported"""
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
class HeaderParsingError(HTTPError):
|
| 325 |
+
"""Raised by assert_header_parsing, but we convert it to a log.warning statement."""
|
| 326 |
+
|
| 327 |
+
def __init__(
|
| 328 |
+
self, defects: list[MessageDefect], unparsed_data: bytes | str | None
|
| 329 |
+
) -> None:
|
| 330 |
+
message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
|
| 331 |
+
super().__init__(message)
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
class UnrewindableBodyError(HTTPError):
|
| 335 |
+
"""urllib3 encountered an error when trying to rewind a body"""
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/fields.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import email.utils
|
| 4 |
+
import mimetypes
|
| 5 |
+
import typing
|
| 6 |
+
|
| 7 |
+
_TYPE_FIELD_VALUE = typing.Union[str, bytes]
|
| 8 |
+
_TYPE_FIELD_VALUE_TUPLE = typing.Union[
|
| 9 |
+
_TYPE_FIELD_VALUE,
|
| 10 |
+
tuple[str, _TYPE_FIELD_VALUE],
|
| 11 |
+
tuple[str, _TYPE_FIELD_VALUE, str],
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def guess_content_type(
|
| 16 |
+
filename: str | None, default: str = "application/octet-stream"
|
| 17 |
+
) -> str:
|
| 18 |
+
"""
|
| 19 |
+
Guess the "Content-Type" of a file.
|
| 20 |
+
|
| 21 |
+
:param filename:
|
| 22 |
+
The filename to guess the "Content-Type" of using :mod:`mimetypes`.
|
| 23 |
+
:param default:
|
| 24 |
+
If no "Content-Type" can be guessed, default to `default`.
|
| 25 |
+
"""
|
| 26 |
+
if filename:
|
| 27 |
+
return mimetypes.guess_type(filename)[0] or default
|
| 28 |
+
return default
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
| 32 |
+
"""
|
| 33 |
+
Helper function to format and quote a single header parameter using the
|
| 34 |
+
strategy defined in RFC 2231.
|
| 35 |
+
|
| 36 |
+
Particularly useful for header parameters which might contain
|
| 37 |
+
non-ASCII values, like file names. This follows
|
| 38 |
+
`RFC 2388 Section 4.4 <https://tools.ietf.org/html/rfc2388#section-4.4>`_.
|
| 39 |
+
|
| 40 |
+
:param name:
|
| 41 |
+
The name of the parameter, a string expected to be ASCII only.
|
| 42 |
+
:param value:
|
| 43 |
+
The value of the parameter, provided as ``bytes`` or `str``.
|
| 44 |
+
:returns:
|
| 45 |
+
An RFC-2231-formatted unicode string.
|
| 46 |
+
|
| 47 |
+
.. deprecated:: 2.0.0
|
| 48 |
+
Will be removed in urllib3 v2.1.0. This is not valid for
|
| 49 |
+
``multipart/form-data`` header parameters.
|
| 50 |
+
"""
|
| 51 |
+
import warnings
|
| 52 |
+
|
| 53 |
+
warnings.warn(
|
| 54 |
+
"'format_header_param_rfc2231' is deprecated and will be "
|
| 55 |
+
"removed in urllib3 v2.1.0. This is not valid for "
|
| 56 |
+
"multipart/form-data header parameters.",
|
| 57 |
+
DeprecationWarning,
|
| 58 |
+
stacklevel=2,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
if isinstance(value, bytes):
|
| 62 |
+
value = value.decode("utf-8")
|
| 63 |
+
|
| 64 |
+
if not any(ch in value for ch in '"\\\r\n'):
|
| 65 |
+
result = f'{name}="{value}"'
|
| 66 |
+
try:
|
| 67 |
+
result.encode("ascii")
|
| 68 |
+
except (UnicodeEncodeError, UnicodeDecodeError):
|
| 69 |
+
pass
|
| 70 |
+
else:
|
| 71 |
+
return result
|
| 72 |
+
|
| 73 |
+
value = email.utils.encode_rfc2231(value, "utf-8")
|
| 74 |
+
value = f"{name}*={value}"
|
| 75 |
+
|
| 76 |
+
return value
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
| 80 |
+
"""
|
| 81 |
+
Format and quote a single multipart header parameter.
|
| 82 |
+
|
| 83 |
+
This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
|
| 84 |
+
the behavior of current browser and curl versions. Values are
|
| 85 |
+
assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
|
| 86 |
+
percent encoded.
|
| 87 |
+
|
| 88 |
+
.. _WHATWG HTML Standard:
|
| 89 |
+
https://html.spec.whatwg.org/multipage/
|
| 90 |
+
form-control-infrastructure.html#multipart-form-data
|
| 91 |
+
|
| 92 |
+
:param name:
|
| 93 |
+
The name of the parameter, an ASCII-only ``str``.
|
| 94 |
+
:param value:
|
| 95 |
+
The value of the parameter, a ``str`` or UTF-8 encoded
|
| 96 |
+
``bytes``.
|
| 97 |
+
:returns:
|
| 98 |
+
A string ``name="value"`` with the escaped value.
|
| 99 |
+
|
| 100 |
+
.. versionchanged:: 2.0.0
|
| 101 |
+
Matches the WHATWG HTML Standard as of 2021/06/10. Control
|
| 102 |
+
characters are no longer percent encoded.
|
| 103 |
+
|
| 104 |
+
.. versionchanged:: 2.0.0
|
| 105 |
+
Renamed from ``format_header_param_html5`` and
|
| 106 |
+
``format_header_param``. The old names will be removed in
|
| 107 |
+
urllib3 v2.1.0.
|
| 108 |
+
"""
|
| 109 |
+
if isinstance(value, bytes):
|
| 110 |
+
value = value.decode("utf-8")
|
| 111 |
+
|
| 112 |
+
# percent encode \n \r "
|
| 113 |
+
value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
|
| 114 |
+
return f'{name}="{value}"'
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
| 118 |
+
"""
|
| 119 |
+
.. deprecated:: 2.0.0
|
| 120 |
+
Renamed to :func:`format_multipart_header_param`. Will be
|
| 121 |
+
removed in urllib3 v2.1.0.
|
| 122 |
+
"""
|
| 123 |
+
import warnings
|
| 124 |
+
|
| 125 |
+
warnings.warn(
|
| 126 |
+
"'format_header_param_html5' has been renamed to "
|
| 127 |
+
"'format_multipart_header_param'. The old name will be "
|
| 128 |
+
"removed in urllib3 v2.1.0.",
|
| 129 |
+
DeprecationWarning,
|
| 130 |
+
stacklevel=2,
|
| 131 |
+
)
|
| 132 |
+
return format_multipart_header_param(name, value)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
| 136 |
+
"""
|
| 137 |
+
.. deprecated:: 2.0.0
|
| 138 |
+
Renamed to :func:`format_multipart_header_param`. Will be
|
| 139 |
+
removed in urllib3 v2.1.0.
|
| 140 |
+
"""
|
| 141 |
+
import warnings
|
| 142 |
+
|
| 143 |
+
warnings.warn(
|
| 144 |
+
"'format_header_param' has been renamed to "
|
| 145 |
+
"'format_multipart_header_param'. The old name will be "
|
| 146 |
+
"removed in urllib3 v2.1.0.",
|
| 147 |
+
DeprecationWarning,
|
| 148 |
+
stacklevel=2,
|
| 149 |
+
)
|
| 150 |
+
return format_multipart_header_param(name, value)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class RequestField:
|
| 154 |
+
"""
|
| 155 |
+
A data container for request body parameters.
|
| 156 |
+
|
| 157 |
+
:param name:
|
| 158 |
+
The name of this request field. Must be unicode.
|
| 159 |
+
:param data:
|
| 160 |
+
The data/value body.
|
| 161 |
+
:param filename:
|
| 162 |
+
An optional filename of the request field. Must be unicode.
|
| 163 |
+
:param headers:
|
| 164 |
+
An optional dict-like object of headers to initially use for the field.
|
| 165 |
+
|
| 166 |
+
.. versionchanged:: 2.0.0
|
| 167 |
+
The ``header_formatter`` parameter is deprecated and will
|
| 168 |
+
be removed in urllib3 v2.1.0.
|
| 169 |
+
"""
|
| 170 |
+
|
| 171 |
+
def __init__(
|
| 172 |
+
self,
|
| 173 |
+
name: str,
|
| 174 |
+
data: _TYPE_FIELD_VALUE,
|
| 175 |
+
filename: str | None = None,
|
| 176 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 177 |
+
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
| 178 |
+
):
|
| 179 |
+
self._name = name
|
| 180 |
+
self._filename = filename
|
| 181 |
+
self.data = data
|
| 182 |
+
self.headers: dict[str, str | None] = {}
|
| 183 |
+
if headers:
|
| 184 |
+
self.headers = dict(headers)
|
| 185 |
+
|
| 186 |
+
if header_formatter is not None:
|
| 187 |
+
import warnings
|
| 188 |
+
|
| 189 |
+
warnings.warn(
|
| 190 |
+
"The 'header_formatter' parameter is deprecated and "
|
| 191 |
+
"will be removed in urllib3 v2.1.0.",
|
| 192 |
+
DeprecationWarning,
|
| 193 |
+
stacklevel=2,
|
| 194 |
+
)
|
| 195 |
+
self.header_formatter = header_formatter
|
| 196 |
+
else:
|
| 197 |
+
self.header_formatter = format_multipart_header_param
|
| 198 |
+
|
| 199 |
+
@classmethod
|
| 200 |
+
def from_tuples(
|
| 201 |
+
cls,
|
| 202 |
+
fieldname: str,
|
| 203 |
+
value: _TYPE_FIELD_VALUE_TUPLE,
|
| 204 |
+
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
| 205 |
+
) -> RequestField:
|
| 206 |
+
"""
|
| 207 |
+
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
|
| 208 |
+
|
| 209 |
+
Supports constructing :class:`~urllib3.fields.RequestField` from
|
| 210 |
+
parameter of key/value strings AND key/filetuple. A filetuple is a
|
| 211 |
+
(filename, data, MIME type) tuple where the MIME type is optional.
|
| 212 |
+
For example::
|
| 213 |
+
|
| 214 |
+
'foo': 'bar',
|
| 215 |
+
'fakefile': ('foofile.txt', 'contents of foofile'),
|
| 216 |
+
'realfile': ('barfile.txt', open('realfile').read()),
|
| 217 |
+
'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'),
|
| 218 |
+
'nonamefile': 'contents of nonamefile field',
|
| 219 |
+
|
| 220 |
+
Field names and filenames must be unicode.
|
| 221 |
+
"""
|
| 222 |
+
filename: str | None
|
| 223 |
+
content_type: str | None
|
| 224 |
+
data: _TYPE_FIELD_VALUE
|
| 225 |
+
|
| 226 |
+
if isinstance(value, tuple):
|
| 227 |
+
if len(value) == 3:
|
| 228 |
+
filename, data, content_type = value
|
| 229 |
+
else:
|
| 230 |
+
filename, data = value
|
| 231 |
+
content_type = guess_content_type(filename)
|
| 232 |
+
else:
|
| 233 |
+
filename = None
|
| 234 |
+
content_type = None
|
| 235 |
+
data = value
|
| 236 |
+
|
| 237 |
+
request_param = cls(
|
| 238 |
+
fieldname, data, filename=filename, header_formatter=header_formatter
|
| 239 |
+
)
|
| 240 |
+
request_param.make_multipart(content_type=content_type)
|
| 241 |
+
|
| 242 |
+
return request_param
|
| 243 |
+
|
| 244 |
+
def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
|
| 245 |
+
"""
|
| 246 |
+
Override this method to change how each multipart header
|
| 247 |
+
parameter is formatted. By default, this calls
|
| 248 |
+
:func:`format_multipart_header_param`.
|
| 249 |
+
|
| 250 |
+
:param name:
|
| 251 |
+
The name of the parameter, an ASCII-only ``str``.
|
| 252 |
+
:param value:
|
| 253 |
+
The value of the parameter, a ``str`` or UTF-8 encoded
|
| 254 |
+
``bytes``.
|
| 255 |
+
|
| 256 |
+
:meta public:
|
| 257 |
+
"""
|
| 258 |
+
return self.header_formatter(name, value)
|
| 259 |
+
|
| 260 |
+
def _render_parts(
|
| 261 |
+
self,
|
| 262 |
+
header_parts: (
|
| 263 |
+
dict[str, _TYPE_FIELD_VALUE | None]
|
| 264 |
+
| typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
|
| 265 |
+
),
|
| 266 |
+
) -> str:
|
| 267 |
+
"""
|
| 268 |
+
Helper function to format and quote a single header.
|
| 269 |
+
|
| 270 |
+
Useful for single headers that are composed of multiple items. E.g.,
|
| 271 |
+
'Content-Disposition' fields.
|
| 272 |
+
|
| 273 |
+
:param header_parts:
|
| 274 |
+
A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
|
| 275 |
+
as `k1="v1"; k2="v2"; ...`.
|
| 276 |
+
"""
|
| 277 |
+
iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
|
| 278 |
+
|
| 279 |
+
parts = []
|
| 280 |
+
if isinstance(header_parts, dict):
|
| 281 |
+
iterable = header_parts.items()
|
| 282 |
+
else:
|
| 283 |
+
iterable = header_parts
|
| 284 |
+
|
| 285 |
+
for name, value in iterable:
|
| 286 |
+
if value is not None:
|
| 287 |
+
parts.append(self._render_part(name, value))
|
| 288 |
+
|
| 289 |
+
return "; ".join(parts)
|
| 290 |
+
|
| 291 |
+
def render_headers(self) -> str:
|
| 292 |
+
"""
|
| 293 |
+
Renders the headers for this request field.
|
| 294 |
+
"""
|
| 295 |
+
lines = []
|
| 296 |
+
|
| 297 |
+
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
|
| 298 |
+
for sort_key in sort_keys:
|
| 299 |
+
if self.headers.get(sort_key, False):
|
| 300 |
+
lines.append(f"{sort_key}: {self.headers[sort_key]}")
|
| 301 |
+
|
| 302 |
+
for header_name, header_value in self.headers.items():
|
| 303 |
+
if header_name not in sort_keys:
|
| 304 |
+
if header_value:
|
| 305 |
+
lines.append(f"{header_name}: {header_value}")
|
| 306 |
+
|
| 307 |
+
lines.append("\r\n")
|
| 308 |
+
return "\r\n".join(lines)
|
| 309 |
+
|
| 310 |
+
def make_multipart(
|
| 311 |
+
self,
|
| 312 |
+
content_disposition: str | None = None,
|
| 313 |
+
content_type: str | None = None,
|
| 314 |
+
content_location: str | None = None,
|
| 315 |
+
) -> None:
|
| 316 |
+
"""
|
| 317 |
+
Makes this request field into a multipart request field.
|
| 318 |
+
|
| 319 |
+
This method overrides "Content-Disposition", "Content-Type" and
|
| 320 |
+
"Content-Location" headers to the request parameter.
|
| 321 |
+
|
| 322 |
+
:param content_disposition:
|
| 323 |
+
The 'Content-Disposition' of the request body. Defaults to 'form-data'
|
| 324 |
+
:param content_type:
|
| 325 |
+
The 'Content-Type' of the request body.
|
| 326 |
+
:param content_location:
|
| 327 |
+
The 'Content-Location' of the request body.
|
| 328 |
+
|
| 329 |
+
"""
|
| 330 |
+
content_disposition = (content_disposition or "form-data") + "; ".join(
|
| 331 |
+
[
|
| 332 |
+
"",
|
| 333 |
+
self._render_parts(
|
| 334 |
+
(("name", self._name), ("filename", self._filename))
|
| 335 |
+
),
|
| 336 |
+
]
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
self.headers["Content-Disposition"] = content_disposition
|
| 340 |
+
self.headers["Content-Type"] = content_type
|
| 341 |
+
self.headers["Content-Location"] = content_location
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/filepost.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import binascii
|
| 4 |
+
import codecs
|
| 5 |
+
import os
|
| 6 |
+
import typing
|
| 7 |
+
from io import BytesIO
|
| 8 |
+
|
| 9 |
+
from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
|
| 10 |
+
|
| 11 |
+
writer = codecs.lookup("utf-8")[3]
|
| 12 |
+
|
| 13 |
+
_TYPE_FIELDS_SEQUENCE = typing.Sequence[
|
| 14 |
+
typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
|
| 15 |
+
]
|
| 16 |
+
_TYPE_FIELDS = typing.Union[
|
| 17 |
+
_TYPE_FIELDS_SEQUENCE,
|
| 18 |
+
typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def choose_boundary() -> str:
|
| 23 |
+
"""
|
| 24 |
+
Our embarrassingly-simple replacement for mimetools.choose_boundary.
|
| 25 |
+
"""
|
| 26 |
+
return binascii.hexlify(os.urandom(16)).decode()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
|
| 30 |
+
"""
|
| 31 |
+
Iterate over fields.
|
| 32 |
+
|
| 33 |
+
Supports list of (k, v) tuples and dicts, and lists of
|
| 34 |
+
:class:`~urllib3.fields.RequestField`.
|
| 35 |
+
|
| 36 |
+
"""
|
| 37 |
+
iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
|
| 38 |
+
|
| 39 |
+
if isinstance(fields, typing.Mapping):
|
| 40 |
+
iterable = fields.items()
|
| 41 |
+
else:
|
| 42 |
+
iterable = fields
|
| 43 |
+
|
| 44 |
+
for field in iterable:
|
| 45 |
+
if isinstance(field, RequestField):
|
| 46 |
+
yield field
|
| 47 |
+
else:
|
| 48 |
+
yield RequestField.from_tuples(*field)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def encode_multipart_formdata(
|
| 52 |
+
fields: _TYPE_FIELDS, boundary: str | None = None
|
| 53 |
+
) -> tuple[bytes, str]:
|
| 54 |
+
"""
|
| 55 |
+
Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
|
| 56 |
+
|
| 57 |
+
:param fields:
|
| 58 |
+
Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
|
| 59 |
+
Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
|
| 60 |
+
|
| 61 |
+
:param boundary:
|
| 62 |
+
If not specified, then a random boundary will be generated using
|
| 63 |
+
:func:`urllib3.filepost.choose_boundary`.
|
| 64 |
+
"""
|
| 65 |
+
body = BytesIO()
|
| 66 |
+
if boundary is None:
|
| 67 |
+
boundary = choose_boundary()
|
| 68 |
+
|
| 69 |
+
for field in iter_field_objects(fields):
|
| 70 |
+
body.write(f"--{boundary}\r\n".encode("latin-1"))
|
| 71 |
+
|
| 72 |
+
writer(body).write(field.render_headers())
|
| 73 |
+
data = field.data
|
| 74 |
+
|
| 75 |
+
if isinstance(data, int):
|
| 76 |
+
data = str(data) # Backwards compatibility
|
| 77 |
+
|
| 78 |
+
if isinstance(data, str):
|
| 79 |
+
writer(body).write(data)
|
| 80 |
+
else:
|
| 81 |
+
body.write(data)
|
| 82 |
+
|
| 83 |
+
body.write(b"\r\n")
|
| 84 |
+
|
| 85 |
+
body.write(f"--{boundary}--\r\n".encode("latin-1"))
|
| 86 |
+
|
| 87 |
+
content_type = f"multipart/form-data; boundary={boundary}"
|
| 88 |
+
|
| 89 |
+
return body.getvalue(), content_type
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__init__.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from importlib.metadata import version
|
| 4 |
+
|
| 5 |
+
__all__ = [
|
| 6 |
+
"inject_into_urllib3",
|
| 7 |
+
"extract_from_urllib3",
|
| 8 |
+
]
|
| 9 |
+
|
| 10 |
+
import typing
|
| 11 |
+
|
| 12 |
+
orig_HTTPSConnection: typing.Any = None
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def inject_into_urllib3() -> None:
|
| 16 |
+
# First check if h2 version is valid
|
| 17 |
+
h2_version = version("h2")
|
| 18 |
+
if not h2_version.startswith("4."):
|
| 19 |
+
raise ImportError(
|
| 20 |
+
"urllib3 v2 supports h2 version 4.x.x, currently "
|
| 21 |
+
f"the 'h2' module is compiled with {h2_version!r}. "
|
| 22 |
+
"See: https://github.com/urllib3/urllib3/issues/3290"
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
# Import here to avoid circular dependencies.
|
| 26 |
+
from .. import connection as urllib3_connection
|
| 27 |
+
from .. import util as urllib3_util
|
| 28 |
+
from ..connectionpool import HTTPSConnectionPool
|
| 29 |
+
from ..util import ssl_ as urllib3_util_ssl
|
| 30 |
+
from .connection import HTTP2Connection
|
| 31 |
+
|
| 32 |
+
global orig_HTTPSConnection
|
| 33 |
+
orig_HTTPSConnection = urllib3_connection.HTTPSConnection
|
| 34 |
+
|
| 35 |
+
HTTPSConnectionPool.ConnectionCls = HTTP2Connection
|
| 36 |
+
urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc]
|
| 37 |
+
|
| 38 |
+
# TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
|
| 39 |
+
urllib3_util.ALPN_PROTOCOLS = ["h2"]
|
| 40 |
+
urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def extract_from_urllib3() -> None:
|
| 44 |
+
from .. import connection as urllib3_connection
|
| 45 |
+
from .. import util as urllib3_util
|
| 46 |
+
from ..connectionpool import HTTPSConnectionPool
|
| 47 |
+
from ..util import ssl_ as urllib3_util_ssl
|
| 48 |
+
|
| 49 |
+
HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
|
| 50 |
+
urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc]
|
| 51 |
+
|
| 52 |
+
urllib3_util.ALPN_PROTOCOLS = ["http/1.1"]
|
| 53 |
+
urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"]
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (1.75 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/__pycache__/probe.cpython-312.pyc
ADDED
|
Binary file (3.66 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/connection.py
ADDED
|
@@ -0,0 +1,356 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import logging
|
| 4 |
+
import re
|
| 5 |
+
import threading
|
| 6 |
+
import types
|
| 7 |
+
import typing
|
| 8 |
+
|
| 9 |
+
import h2.config
|
| 10 |
+
import h2.connection
|
| 11 |
+
import h2.events
|
| 12 |
+
|
| 13 |
+
from .._base_connection import _TYPE_BODY
|
| 14 |
+
from .._collections import HTTPHeaderDict
|
| 15 |
+
from ..connection import HTTPSConnection, _get_default_user_agent
|
| 16 |
+
from ..exceptions import ConnectionError
|
| 17 |
+
from ..response import BaseHTTPResponse
|
| 18 |
+
|
| 19 |
+
orig_HTTPSConnection = HTTPSConnection
|
| 20 |
+
|
| 21 |
+
T = typing.TypeVar("T")
|
| 22 |
+
|
| 23 |
+
log = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$")
|
| 26 |
+
RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$")
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _is_legal_header_name(name: bytes) -> bool:
|
| 30 |
+
"""
|
| 31 |
+
"An implementation that validates fields according to the definitions in Sections
|
| 32 |
+
5.1 and 5.5 of [HTTP] only needs an additional check that field names do not
|
| 33 |
+
include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
|
| 34 |
+
|
| 35 |
+
`http.client._is_legal_header_name` does not validate the field name according to the
|
| 36 |
+
HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters.
|
| 37 |
+
|
| 38 |
+
This does not allow for the `:` character in the header name, so should not
|
| 39 |
+
be used to validate pseudo-headers.
|
| 40 |
+
"""
|
| 41 |
+
return bool(RE_IS_LEGAL_HEADER_NAME.match(name))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _is_illegal_header_value(value: bytes) -> bool:
|
| 45 |
+
"""
|
| 46 |
+
"A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed
|
| 47 |
+
(ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field
|
| 48 |
+
value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB,
|
| 49 |
+
0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
|
| 50 |
+
"""
|
| 51 |
+
return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class _LockedObject(typing.Generic[T]):
|
| 55 |
+
"""
|
| 56 |
+
A wrapper class that hides a specific object behind a lock.
|
| 57 |
+
The goal here is to provide a simple way to protect access to an object
|
| 58 |
+
that cannot safely be simultaneously accessed from multiple threads. The
|
| 59 |
+
intended use of this class is simple: take hold of it with a context
|
| 60 |
+
manager, which returns the protected object.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
__slots__ = (
|
| 64 |
+
"lock",
|
| 65 |
+
"_obj",
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
def __init__(self, obj: T):
|
| 69 |
+
self.lock = threading.RLock()
|
| 70 |
+
self._obj = obj
|
| 71 |
+
|
| 72 |
+
def __enter__(self) -> T:
|
| 73 |
+
self.lock.acquire()
|
| 74 |
+
return self._obj
|
| 75 |
+
|
| 76 |
+
def __exit__(
|
| 77 |
+
self,
|
| 78 |
+
exc_type: type[BaseException] | None,
|
| 79 |
+
exc_val: BaseException | None,
|
| 80 |
+
exc_tb: types.TracebackType | None,
|
| 81 |
+
) -> None:
|
| 82 |
+
self.lock.release()
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class HTTP2Connection(HTTPSConnection):
|
| 86 |
+
def __init__(
|
| 87 |
+
self, host: str, port: int | None = None, **kwargs: typing.Any
|
| 88 |
+
) -> None:
|
| 89 |
+
self._h2_conn = self._new_h2_conn()
|
| 90 |
+
self._h2_stream: int | None = None
|
| 91 |
+
self._headers: list[tuple[bytes, bytes]] = []
|
| 92 |
+
|
| 93 |
+
if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive:
|
| 94 |
+
raise NotImplementedError("Proxies aren't supported with HTTP/2")
|
| 95 |
+
|
| 96 |
+
super().__init__(host, port, **kwargs)
|
| 97 |
+
|
| 98 |
+
if self._tunnel_host is not None:
|
| 99 |
+
raise NotImplementedError("Tunneling isn't supported with HTTP/2")
|
| 100 |
+
|
| 101 |
+
def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]:
|
| 102 |
+
config = h2.config.H2Configuration(client_side=True)
|
| 103 |
+
return _LockedObject(h2.connection.H2Connection(config=config))
|
| 104 |
+
|
| 105 |
+
def connect(self) -> None:
|
| 106 |
+
super().connect()
|
| 107 |
+
with self._h2_conn as conn:
|
| 108 |
+
conn.initiate_connection()
|
| 109 |
+
if data_to_send := conn.data_to_send():
|
| 110 |
+
self.sock.sendall(data_to_send)
|
| 111 |
+
|
| 112 |
+
def putrequest( # type: ignore[override]
|
| 113 |
+
self,
|
| 114 |
+
method: str,
|
| 115 |
+
url: str,
|
| 116 |
+
**kwargs: typing.Any,
|
| 117 |
+
) -> None:
|
| 118 |
+
"""putrequest
|
| 119 |
+
This deviates from the HTTPConnection method signature since we never need to override
|
| 120 |
+
sending accept-encoding headers or the host header.
|
| 121 |
+
"""
|
| 122 |
+
if "skip_host" in kwargs:
|
| 123 |
+
raise NotImplementedError("`skip_host` isn't supported")
|
| 124 |
+
if "skip_accept_encoding" in kwargs:
|
| 125 |
+
raise NotImplementedError("`skip_accept_encoding` isn't supported")
|
| 126 |
+
|
| 127 |
+
self._request_url = url or "/"
|
| 128 |
+
self._validate_path(url) # type: ignore[attr-defined]
|
| 129 |
+
|
| 130 |
+
if ":" in self.host:
|
| 131 |
+
authority = f"[{self.host}]:{self.port or 443}"
|
| 132 |
+
else:
|
| 133 |
+
authority = f"{self.host}:{self.port or 443}"
|
| 134 |
+
|
| 135 |
+
self._headers.append((b":scheme", b"https"))
|
| 136 |
+
self._headers.append((b":method", method.encode()))
|
| 137 |
+
self._headers.append((b":authority", authority.encode()))
|
| 138 |
+
self._headers.append((b":path", url.encode()))
|
| 139 |
+
|
| 140 |
+
with self._h2_conn as conn:
|
| 141 |
+
self._h2_stream = conn.get_next_available_stream_id()
|
| 142 |
+
|
| 143 |
+
def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override]
|
| 144 |
+
# TODO SKIPPABLE_HEADERS from urllib3 are ignored.
|
| 145 |
+
header = header.encode() if isinstance(header, str) else header
|
| 146 |
+
header = header.lower() # A lot of upstream code uses capitalized headers.
|
| 147 |
+
if not _is_legal_header_name(header):
|
| 148 |
+
raise ValueError(f"Illegal header name {str(header)}")
|
| 149 |
+
|
| 150 |
+
for value in values:
|
| 151 |
+
value = value.encode() if isinstance(value, str) else value
|
| 152 |
+
if _is_illegal_header_value(value):
|
| 153 |
+
raise ValueError(f"Illegal header value {str(value)}")
|
| 154 |
+
self._headers.append((header, value))
|
| 155 |
+
|
| 156 |
+
def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override]
|
| 157 |
+
if self._h2_stream is None:
|
| 158 |
+
raise ConnectionError("Must call `putrequest` first.")
|
| 159 |
+
|
| 160 |
+
with self._h2_conn as conn:
|
| 161 |
+
conn.send_headers(
|
| 162 |
+
stream_id=self._h2_stream,
|
| 163 |
+
headers=self._headers,
|
| 164 |
+
end_stream=(message_body is None),
|
| 165 |
+
)
|
| 166 |
+
if data_to_send := conn.data_to_send():
|
| 167 |
+
self.sock.sendall(data_to_send)
|
| 168 |
+
self._headers = [] # Reset headers for the next request.
|
| 169 |
+
|
| 170 |
+
def send(self, data: typing.Any) -> None:
|
| 171 |
+
"""Send data to the server.
|
| 172 |
+
`data` can be: `str`, `bytes`, an iterable, or file-like objects
|
| 173 |
+
that support a .read() method.
|
| 174 |
+
"""
|
| 175 |
+
if self._h2_stream is None:
|
| 176 |
+
raise ConnectionError("Must call `putrequest` first.")
|
| 177 |
+
|
| 178 |
+
with self._h2_conn as conn:
|
| 179 |
+
if data_to_send := conn.data_to_send():
|
| 180 |
+
self.sock.sendall(data_to_send)
|
| 181 |
+
|
| 182 |
+
if hasattr(data, "read"): # file-like objects
|
| 183 |
+
while True:
|
| 184 |
+
chunk = data.read(self.blocksize)
|
| 185 |
+
if not chunk:
|
| 186 |
+
break
|
| 187 |
+
if isinstance(chunk, str):
|
| 188 |
+
chunk = chunk.encode()
|
| 189 |
+
conn.send_data(self._h2_stream, chunk, end_stream=False)
|
| 190 |
+
if data_to_send := conn.data_to_send():
|
| 191 |
+
self.sock.sendall(data_to_send)
|
| 192 |
+
conn.end_stream(self._h2_stream)
|
| 193 |
+
return
|
| 194 |
+
|
| 195 |
+
if isinstance(data, str): # str -> bytes
|
| 196 |
+
data = data.encode()
|
| 197 |
+
|
| 198 |
+
try:
|
| 199 |
+
if isinstance(data, bytes):
|
| 200 |
+
conn.send_data(self._h2_stream, data, end_stream=True)
|
| 201 |
+
if data_to_send := conn.data_to_send():
|
| 202 |
+
self.sock.sendall(data_to_send)
|
| 203 |
+
else:
|
| 204 |
+
for chunk in data:
|
| 205 |
+
conn.send_data(self._h2_stream, chunk, end_stream=False)
|
| 206 |
+
if data_to_send := conn.data_to_send():
|
| 207 |
+
self.sock.sendall(data_to_send)
|
| 208 |
+
conn.end_stream(self._h2_stream)
|
| 209 |
+
except TypeError:
|
| 210 |
+
raise TypeError(
|
| 211 |
+
"`data` should be str, bytes, iterable, or file. got %r"
|
| 212 |
+
% type(data)
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
def set_tunnel(
|
| 216 |
+
self,
|
| 217 |
+
host: str,
|
| 218 |
+
port: int | None = None,
|
| 219 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 220 |
+
scheme: str = "http",
|
| 221 |
+
) -> None:
|
| 222 |
+
raise NotImplementedError(
|
| 223 |
+
"HTTP/2 does not support setting up a tunnel through a proxy"
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
def getresponse( # type: ignore[override]
|
| 227 |
+
self,
|
| 228 |
+
) -> HTTP2Response:
|
| 229 |
+
status = None
|
| 230 |
+
data = bytearray()
|
| 231 |
+
with self._h2_conn as conn:
|
| 232 |
+
end_stream = False
|
| 233 |
+
while not end_stream:
|
| 234 |
+
# TODO: Arbitrary read value.
|
| 235 |
+
if received_data := self.sock.recv(65535):
|
| 236 |
+
events = conn.receive_data(received_data)
|
| 237 |
+
for event in events:
|
| 238 |
+
if isinstance(event, h2.events.ResponseReceived):
|
| 239 |
+
headers = HTTPHeaderDict()
|
| 240 |
+
for header, value in event.headers:
|
| 241 |
+
if header == b":status":
|
| 242 |
+
status = int(value.decode())
|
| 243 |
+
else:
|
| 244 |
+
headers.add(
|
| 245 |
+
header.decode("ascii"), value.decode("ascii")
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
elif isinstance(event, h2.events.DataReceived):
|
| 249 |
+
data += event.data
|
| 250 |
+
conn.acknowledge_received_data(
|
| 251 |
+
event.flow_controlled_length, event.stream_id
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
elif isinstance(event, h2.events.StreamEnded):
|
| 255 |
+
end_stream = True
|
| 256 |
+
|
| 257 |
+
if data_to_send := conn.data_to_send():
|
| 258 |
+
self.sock.sendall(data_to_send)
|
| 259 |
+
|
| 260 |
+
assert status is not None
|
| 261 |
+
return HTTP2Response(
|
| 262 |
+
status=status,
|
| 263 |
+
headers=headers,
|
| 264 |
+
request_url=self._request_url,
|
| 265 |
+
data=bytes(data),
|
| 266 |
+
)
|
| 267 |
+
|
| 268 |
+
def request( # type: ignore[override]
|
| 269 |
+
self,
|
| 270 |
+
method: str,
|
| 271 |
+
url: str,
|
| 272 |
+
body: _TYPE_BODY | None = None,
|
| 273 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 274 |
+
*,
|
| 275 |
+
preload_content: bool = True,
|
| 276 |
+
decode_content: bool = True,
|
| 277 |
+
enforce_content_length: bool = True,
|
| 278 |
+
**kwargs: typing.Any,
|
| 279 |
+
) -> None:
|
| 280 |
+
"""Send an HTTP/2 request"""
|
| 281 |
+
if "chunked" in kwargs:
|
| 282 |
+
# TODO this is often present from upstream.
|
| 283 |
+
# raise NotImplementedError("`chunked` isn't supported with HTTP/2")
|
| 284 |
+
pass
|
| 285 |
+
|
| 286 |
+
if self.sock is not None:
|
| 287 |
+
self.sock.settimeout(self.timeout)
|
| 288 |
+
|
| 289 |
+
self.putrequest(method, url)
|
| 290 |
+
|
| 291 |
+
headers = headers or {}
|
| 292 |
+
for k, v in headers.items():
|
| 293 |
+
if k.lower() == "transfer-encoding" and v == "chunked":
|
| 294 |
+
continue
|
| 295 |
+
else:
|
| 296 |
+
self.putheader(k, v)
|
| 297 |
+
|
| 298 |
+
if b"user-agent" not in dict(self._headers):
|
| 299 |
+
self.putheader(b"user-agent", _get_default_user_agent())
|
| 300 |
+
|
| 301 |
+
if body:
|
| 302 |
+
self.endheaders(message_body=body)
|
| 303 |
+
self.send(body)
|
| 304 |
+
else:
|
| 305 |
+
self.endheaders()
|
| 306 |
+
|
| 307 |
+
def close(self) -> None:
|
| 308 |
+
with self._h2_conn as conn:
|
| 309 |
+
try:
|
| 310 |
+
conn.close_connection()
|
| 311 |
+
if data := conn.data_to_send():
|
| 312 |
+
self.sock.sendall(data)
|
| 313 |
+
except Exception:
|
| 314 |
+
pass
|
| 315 |
+
|
| 316 |
+
# Reset all our HTTP/2 connection state.
|
| 317 |
+
self._h2_conn = self._new_h2_conn()
|
| 318 |
+
self._h2_stream = None
|
| 319 |
+
self._headers = []
|
| 320 |
+
|
| 321 |
+
super().close()
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
class HTTP2Response(BaseHTTPResponse):
|
| 325 |
+
# TODO: This is a woefully incomplete response object, but works for non-streaming.
|
| 326 |
+
def __init__(
|
| 327 |
+
self,
|
| 328 |
+
status: int,
|
| 329 |
+
headers: HTTPHeaderDict,
|
| 330 |
+
request_url: str,
|
| 331 |
+
data: bytes,
|
| 332 |
+
decode_content: bool = False, # TODO: support decoding
|
| 333 |
+
) -> None:
|
| 334 |
+
super().__init__(
|
| 335 |
+
status=status,
|
| 336 |
+
headers=headers,
|
| 337 |
+
# Following CPython, we map HTTP versions to major * 10 + minor integers
|
| 338 |
+
version=20,
|
| 339 |
+
version_string="HTTP/2",
|
| 340 |
+
# No reason phrase in HTTP/2
|
| 341 |
+
reason=None,
|
| 342 |
+
decode_content=decode_content,
|
| 343 |
+
request_url=request_url,
|
| 344 |
+
)
|
| 345 |
+
self._data = data
|
| 346 |
+
self.length_remaining = 0
|
| 347 |
+
|
| 348 |
+
@property
|
| 349 |
+
def data(self) -> bytes:
|
| 350 |
+
return self._data
|
| 351 |
+
|
| 352 |
+
def get_redirect_location(self) -> None:
|
| 353 |
+
return None
|
| 354 |
+
|
| 355 |
+
def close(self) -> None:
|
| 356 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/http2/probe.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import threading
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class _HTTP2ProbeCache:
|
| 7 |
+
__slots__ = (
|
| 8 |
+
"_lock",
|
| 9 |
+
"_cache_locks",
|
| 10 |
+
"_cache_values",
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
def __init__(self) -> None:
|
| 14 |
+
self._lock = threading.Lock()
|
| 15 |
+
self._cache_locks: dict[tuple[str, int], threading.RLock] = {}
|
| 16 |
+
self._cache_values: dict[tuple[str, int], bool | None] = {}
|
| 17 |
+
|
| 18 |
+
def acquire_and_get(self, host: str, port: int) -> bool | None:
|
| 19 |
+
# By the end of this block we know that
|
| 20 |
+
# _cache_[values,locks] is available.
|
| 21 |
+
value = None
|
| 22 |
+
with self._lock:
|
| 23 |
+
key = (host, port)
|
| 24 |
+
try:
|
| 25 |
+
value = self._cache_values[key]
|
| 26 |
+
# If it's a known value we return right away.
|
| 27 |
+
if value is not None:
|
| 28 |
+
return value
|
| 29 |
+
except KeyError:
|
| 30 |
+
self._cache_locks[key] = threading.RLock()
|
| 31 |
+
self._cache_values[key] = None
|
| 32 |
+
|
| 33 |
+
# If the value is unknown, we acquire the lock to signal
|
| 34 |
+
# to the requesting thread that the probe is in progress
|
| 35 |
+
# or that the current thread needs to return their findings.
|
| 36 |
+
key_lock = self._cache_locks[key]
|
| 37 |
+
key_lock.acquire()
|
| 38 |
+
try:
|
| 39 |
+
# If the by the time we get the lock the value has been
|
| 40 |
+
# updated we want to return the updated value.
|
| 41 |
+
value = self._cache_values[key]
|
| 42 |
+
|
| 43 |
+
# In case an exception like KeyboardInterrupt is raised here.
|
| 44 |
+
except BaseException as e: # Defensive:
|
| 45 |
+
assert not isinstance(e, KeyError) # KeyError shouldn't be possible.
|
| 46 |
+
key_lock.release()
|
| 47 |
+
raise
|
| 48 |
+
|
| 49 |
+
return value
|
| 50 |
+
|
| 51 |
+
def set_and_release(
|
| 52 |
+
self, host: str, port: int, supports_http2: bool | None
|
| 53 |
+
) -> None:
|
| 54 |
+
key = (host, port)
|
| 55 |
+
key_lock = self._cache_locks[key]
|
| 56 |
+
with key_lock: # Uses an RLock, so can be locked again from same thread.
|
| 57 |
+
if supports_http2 is None and self._cache_values[key] is not None:
|
| 58 |
+
raise ValueError(
|
| 59 |
+
"Cannot reset HTTP/2 support for origin after value has been set."
|
| 60 |
+
) # Defensive: not expected in normal usage
|
| 61 |
+
|
| 62 |
+
self._cache_values[key] = supports_http2
|
| 63 |
+
key_lock.release()
|
| 64 |
+
|
| 65 |
+
def _values(self) -> dict[tuple[str, int], bool | None]:
|
| 66 |
+
"""This function is for testing purposes only. Gets the current state of the probe cache"""
|
| 67 |
+
with self._lock:
|
| 68 |
+
return {k: v for k, v in self._cache_values.items()}
|
| 69 |
+
|
| 70 |
+
def _reset(self) -> None:
|
| 71 |
+
"""This function is for testing purposes only. Reset the cache values"""
|
| 72 |
+
with self._lock:
|
| 73 |
+
self._cache_locks = {}
|
| 74 |
+
self._cache_values = {}
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
_HTTP2_PROBE_CACHE = _HTTP2ProbeCache()
|
| 78 |
+
|
| 79 |
+
set_and_release = _HTTP2_PROBE_CACHE.set_and_release
|
| 80 |
+
acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get
|
| 81 |
+
_values = _HTTP2_PROBE_CACHE._values
|
| 82 |
+
_reset = _HTTP2_PROBE_CACHE._reset
|
| 83 |
+
|
| 84 |
+
__all__ = [
|
| 85 |
+
"set_and_release",
|
| 86 |
+
"acquire_and_get",
|
| 87 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/poolmanager.py
ADDED
|
@@ -0,0 +1,651 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import functools
|
| 4 |
+
import logging
|
| 5 |
+
import typing
|
| 6 |
+
import warnings
|
| 7 |
+
from types import TracebackType
|
| 8 |
+
from urllib.parse import urljoin
|
| 9 |
+
|
| 10 |
+
from ._collections import HTTPHeaderDict, RecentlyUsedContainer
|
| 11 |
+
from ._request_methods import RequestMethods
|
| 12 |
+
from .connection import ProxyConfig
|
| 13 |
+
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
|
| 14 |
+
from .exceptions import (
|
| 15 |
+
LocationValueError,
|
| 16 |
+
MaxRetryError,
|
| 17 |
+
ProxySchemeUnknown,
|
| 18 |
+
URLSchemeUnknown,
|
| 19 |
+
)
|
| 20 |
+
from .response import BaseHTTPResponse
|
| 21 |
+
from .util.connection import _TYPE_SOCKET_OPTIONS
|
| 22 |
+
from .util.proxy import connection_requires_http_tunnel
|
| 23 |
+
from .util.retry import Retry
|
| 24 |
+
from .util.timeout import Timeout
|
| 25 |
+
from .util.url import Url, parse_url
|
| 26 |
+
|
| 27 |
+
if typing.TYPE_CHECKING:
|
| 28 |
+
import ssl
|
| 29 |
+
|
| 30 |
+
from typing_extensions import Self
|
| 31 |
+
|
| 32 |
+
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
log = logging.getLogger(__name__)
|
| 36 |
+
|
| 37 |
+
SSL_KEYWORDS = (
|
| 38 |
+
"key_file",
|
| 39 |
+
"cert_file",
|
| 40 |
+
"cert_reqs",
|
| 41 |
+
"ca_certs",
|
| 42 |
+
"ca_cert_data",
|
| 43 |
+
"ssl_version",
|
| 44 |
+
"ssl_minimum_version",
|
| 45 |
+
"ssl_maximum_version",
|
| 46 |
+
"ca_cert_dir",
|
| 47 |
+
"ssl_context",
|
| 48 |
+
"key_password",
|
| 49 |
+
"server_hostname",
|
| 50 |
+
)
|
| 51 |
+
# Default value for `blocksize` - a new parameter introduced to
|
| 52 |
+
# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
|
| 53 |
+
_DEFAULT_BLOCKSIZE = 16384
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class PoolKey(typing.NamedTuple):
|
| 57 |
+
"""
|
| 58 |
+
All known keyword arguments that could be provided to the pool manager, its
|
| 59 |
+
pools, or the underlying connections.
|
| 60 |
+
|
| 61 |
+
All custom key schemes should include the fields in this key at a minimum.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
key_scheme: str
|
| 65 |
+
key_host: str
|
| 66 |
+
key_port: int | None
|
| 67 |
+
key_timeout: Timeout | float | int | None
|
| 68 |
+
key_retries: Retry | bool | int | None
|
| 69 |
+
key_block: bool | None
|
| 70 |
+
key_source_address: tuple[str, int] | None
|
| 71 |
+
key_key_file: str | None
|
| 72 |
+
key_key_password: str | None
|
| 73 |
+
key_cert_file: str | None
|
| 74 |
+
key_cert_reqs: str | None
|
| 75 |
+
key_ca_certs: str | None
|
| 76 |
+
key_ca_cert_data: str | bytes | None
|
| 77 |
+
key_ssl_version: int | str | None
|
| 78 |
+
key_ssl_minimum_version: ssl.TLSVersion | None
|
| 79 |
+
key_ssl_maximum_version: ssl.TLSVersion | None
|
| 80 |
+
key_ca_cert_dir: str | None
|
| 81 |
+
key_ssl_context: ssl.SSLContext | None
|
| 82 |
+
key_maxsize: int | None
|
| 83 |
+
key_headers: frozenset[tuple[str, str]] | None
|
| 84 |
+
key__proxy: Url | None
|
| 85 |
+
key__proxy_headers: frozenset[tuple[str, str]] | None
|
| 86 |
+
key__proxy_config: ProxyConfig | None
|
| 87 |
+
key_socket_options: _TYPE_SOCKET_OPTIONS | None
|
| 88 |
+
key__socks_options: frozenset[tuple[str, str]] | None
|
| 89 |
+
key_assert_hostname: bool | str | None
|
| 90 |
+
key_assert_fingerprint: str | None
|
| 91 |
+
key_server_hostname: str | None
|
| 92 |
+
key_blocksize: int | None
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _default_key_normalizer(
|
| 96 |
+
key_class: type[PoolKey], request_context: dict[str, typing.Any]
|
| 97 |
+
) -> PoolKey:
|
| 98 |
+
"""
|
| 99 |
+
Create a pool key out of a request context dictionary.
|
| 100 |
+
|
| 101 |
+
According to RFC 3986, both the scheme and host are case-insensitive.
|
| 102 |
+
Therefore, this function normalizes both before constructing the pool
|
| 103 |
+
key for an HTTPS request. If you wish to change this behaviour, provide
|
| 104 |
+
alternate callables to ``key_fn_by_scheme``.
|
| 105 |
+
|
| 106 |
+
:param key_class:
|
| 107 |
+
The class to use when constructing the key. This should be a namedtuple
|
| 108 |
+
with the ``scheme`` and ``host`` keys at a minimum.
|
| 109 |
+
:type key_class: namedtuple
|
| 110 |
+
:param request_context:
|
| 111 |
+
A dictionary-like object that contain the context for a request.
|
| 112 |
+
:type request_context: dict
|
| 113 |
+
|
| 114 |
+
:return: A namedtuple that can be used as a connection pool key.
|
| 115 |
+
:rtype: PoolKey
|
| 116 |
+
"""
|
| 117 |
+
# Since we mutate the dictionary, make a copy first
|
| 118 |
+
context = request_context.copy()
|
| 119 |
+
context["scheme"] = context["scheme"].lower()
|
| 120 |
+
context["host"] = context["host"].lower()
|
| 121 |
+
|
| 122 |
+
# These are both dictionaries and need to be transformed into frozensets
|
| 123 |
+
for key in ("headers", "_proxy_headers", "_socks_options"):
|
| 124 |
+
if key in context and context[key] is not None:
|
| 125 |
+
context[key] = frozenset(context[key].items())
|
| 126 |
+
|
| 127 |
+
# The socket_options key may be a list and needs to be transformed into a
|
| 128 |
+
# tuple.
|
| 129 |
+
socket_opts = context.get("socket_options")
|
| 130 |
+
if socket_opts is not None:
|
| 131 |
+
context["socket_options"] = tuple(socket_opts)
|
| 132 |
+
|
| 133 |
+
# Map the kwargs to the names in the namedtuple - this is necessary since
|
| 134 |
+
# namedtuples can't have fields starting with '_'.
|
| 135 |
+
for key in list(context.keys()):
|
| 136 |
+
context["key_" + key] = context.pop(key)
|
| 137 |
+
|
| 138 |
+
# Default to ``None`` for keys missing from the context
|
| 139 |
+
for field in key_class._fields:
|
| 140 |
+
if field not in context:
|
| 141 |
+
context[field] = None
|
| 142 |
+
|
| 143 |
+
# Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
|
| 144 |
+
if context.get("key_blocksize") is None:
|
| 145 |
+
context["key_blocksize"] = _DEFAULT_BLOCKSIZE
|
| 146 |
+
|
| 147 |
+
return key_class(**context)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
#: A dictionary that maps a scheme to a callable that creates a pool key.
|
| 151 |
+
#: This can be used to alter the way pool keys are constructed, if desired.
|
| 152 |
+
#: Each PoolManager makes a copy of this dictionary so they can be configured
|
| 153 |
+
#: globally here, or individually on the instance.
|
| 154 |
+
key_fn_by_scheme = {
|
| 155 |
+
"http": functools.partial(_default_key_normalizer, PoolKey),
|
| 156 |
+
"https": functools.partial(_default_key_normalizer, PoolKey),
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class PoolManager(RequestMethods):
|
| 163 |
+
"""
|
| 164 |
+
Allows for arbitrary requests while transparently keeping track of
|
| 165 |
+
necessary connection pools for you.
|
| 166 |
+
|
| 167 |
+
:param num_pools:
|
| 168 |
+
Number of connection pools to cache before discarding the least
|
| 169 |
+
recently used pool.
|
| 170 |
+
|
| 171 |
+
:param headers:
|
| 172 |
+
Headers to include with all requests, unless other headers are given
|
| 173 |
+
explicitly.
|
| 174 |
+
|
| 175 |
+
:param \\**connection_pool_kw:
|
| 176 |
+
Additional parameters are used to create fresh
|
| 177 |
+
:class:`urllib3.connectionpool.ConnectionPool` instances.
|
| 178 |
+
|
| 179 |
+
Example:
|
| 180 |
+
|
| 181 |
+
.. code-block:: python
|
| 182 |
+
|
| 183 |
+
import urllib3
|
| 184 |
+
|
| 185 |
+
http = urllib3.PoolManager(num_pools=2)
|
| 186 |
+
|
| 187 |
+
resp1 = http.request("GET", "https://google.com/")
|
| 188 |
+
resp2 = http.request("GET", "https://google.com/mail")
|
| 189 |
+
resp3 = http.request("GET", "https://yahoo.com/")
|
| 190 |
+
|
| 191 |
+
print(len(http.pools))
|
| 192 |
+
# 2
|
| 193 |
+
|
| 194 |
+
"""
|
| 195 |
+
|
| 196 |
+
proxy: Url | None = None
|
| 197 |
+
proxy_config: ProxyConfig | None = None
|
| 198 |
+
|
| 199 |
+
def __init__(
|
| 200 |
+
self,
|
| 201 |
+
num_pools: int = 10,
|
| 202 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 203 |
+
**connection_pool_kw: typing.Any,
|
| 204 |
+
) -> None:
|
| 205 |
+
super().__init__(headers)
|
| 206 |
+
# PoolManager handles redirects itself in PoolManager.urlopen().
|
| 207 |
+
# It always passes redirect=False to the underlying connection pool to
|
| 208 |
+
# suppress per-pool redirect handling. If the user supplied a non-Retry
|
| 209 |
+
# value (int/bool/etc) for retries and we let the pool normalize it
|
| 210 |
+
# while redirect=False, the resulting Retry object would have redirect
|
| 211 |
+
# handling disabled, which can interfere with PoolManager's own
|
| 212 |
+
# redirect logic. Normalize here so redirects remain governed solely by
|
| 213 |
+
# PoolManager logic.
|
| 214 |
+
if "retries" in connection_pool_kw:
|
| 215 |
+
retries = connection_pool_kw["retries"]
|
| 216 |
+
if not isinstance(retries, Retry):
|
| 217 |
+
retries = Retry.from_int(retries)
|
| 218 |
+
connection_pool_kw = connection_pool_kw.copy()
|
| 219 |
+
connection_pool_kw["retries"] = retries
|
| 220 |
+
self.connection_pool_kw = connection_pool_kw
|
| 221 |
+
|
| 222 |
+
self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
|
| 223 |
+
self.pools = RecentlyUsedContainer(num_pools)
|
| 224 |
+
|
| 225 |
+
# Locally set the pool classes and keys so other PoolManagers can
|
| 226 |
+
# override them.
|
| 227 |
+
self.pool_classes_by_scheme = pool_classes_by_scheme
|
| 228 |
+
self.key_fn_by_scheme = key_fn_by_scheme.copy()
|
| 229 |
+
|
| 230 |
+
def __enter__(self) -> Self:
|
| 231 |
+
return self
|
| 232 |
+
|
| 233 |
+
def __exit__(
|
| 234 |
+
self,
|
| 235 |
+
exc_type: type[BaseException] | None,
|
| 236 |
+
exc_val: BaseException | None,
|
| 237 |
+
exc_tb: TracebackType | None,
|
| 238 |
+
) -> typing.Literal[False]:
|
| 239 |
+
self.clear()
|
| 240 |
+
# Return False to re-raise any potential exceptions
|
| 241 |
+
return False
|
| 242 |
+
|
| 243 |
+
def _new_pool(
|
| 244 |
+
self,
|
| 245 |
+
scheme: str,
|
| 246 |
+
host: str,
|
| 247 |
+
port: int,
|
| 248 |
+
request_context: dict[str, typing.Any] | None = None,
|
| 249 |
+
) -> HTTPConnectionPool:
|
| 250 |
+
"""
|
| 251 |
+
Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
|
| 252 |
+
any additional pool keyword arguments.
|
| 253 |
+
|
| 254 |
+
If ``request_context`` is provided, it is provided as keyword arguments
|
| 255 |
+
to the pool class used. This method is used to actually create the
|
| 256 |
+
connection pools handed out by :meth:`connection_from_url` and
|
| 257 |
+
companion methods. It is intended to be overridden for customization.
|
| 258 |
+
"""
|
| 259 |
+
pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
|
| 260 |
+
if request_context is None:
|
| 261 |
+
request_context = self.connection_pool_kw.copy()
|
| 262 |
+
|
| 263 |
+
# Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
|
| 264 |
+
# set to 'None' in the request_context.
|
| 265 |
+
if request_context.get("blocksize") is None:
|
| 266 |
+
request_context["blocksize"] = _DEFAULT_BLOCKSIZE
|
| 267 |
+
|
| 268 |
+
# Although the context has everything necessary to create the pool,
|
| 269 |
+
# this function has historically only used the scheme, host, and port
|
| 270 |
+
# in the positional args. When an API change is acceptable these can
|
| 271 |
+
# be removed.
|
| 272 |
+
for key in ("scheme", "host", "port"):
|
| 273 |
+
request_context.pop(key, None)
|
| 274 |
+
|
| 275 |
+
if scheme == "http":
|
| 276 |
+
for kw in SSL_KEYWORDS:
|
| 277 |
+
request_context.pop(kw, None)
|
| 278 |
+
|
| 279 |
+
return pool_cls(host, port, **request_context)
|
| 280 |
+
|
| 281 |
+
def clear(self) -> None:
|
| 282 |
+
"""
|
| 283 |
+
Empty our store of pools and direct them all to close.
|
| 284 |
+
|
| 285 |
+
This will not affect in-flight connections, but they will not be
|
| 286 |
+
re-used after completion.
|
| 287 |
+
"""
|
| 288 |
+
self.pools.clear()
|
| 289 |
+
|
| 290 |
+
def connection_from_host(
|
| 291 |
+
self,
|
| 292 |
+
host: str | None,
|
| 293 |
+
port: int | None = None,
|
| 294 |
+
scheme: str | None = "http",
|
| 295 |
+
pool_kwargs: dict[str, typing.Any] | None = None,
|
| 296 |
+
) -> HTTPConnectionPool:
|
| 297 |
+
"""
|
| 298 |
+
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
|
| 299 |
+
|
| 300 |
+
If ``port`` isn't given, it will be derived from the ``scheme`` using
|
| 301 |
+
``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
|
| 302 |
+
provided, it is merged with the instance's ``connection_pool_kw``
|
| 303 |
+
variable and used to create the new connection pool, if one is
|
| 304 |
+
needed.
|
| 305 |
+
"""
|
| 306 |
+
|
| 307 |
+
if not host:
|
| 308 |
+
raise LocationValueError("No host specified.")
|
| 309 |
+
|
| 310 |
+
request_context = self._merge_pool_kwargs(pool_kwargs)
|
| 311 |
+
request_context["scheme"] = scheme or "http"
|
| 312 |
+
if not port:
|
| 313 |
+
port = port_by_scheme.get(request_context["scheme"].lower(), 80)
|
| 314 |
+
request_context["port"] = port
|
| 315 |
+
request_context["host"] = host
|
| 316 |
+
|
| 317 |
+
return self.connection_from_context(request_context)
|
| 318 |
+
|
| 319 |
+
def connection_from_context(
|
| 320 |
+
self, request_context: dict[str, typing.Any]
|
| 321 |
+
) -> HTTPConnectionPool:
|
| 322 |
+
"""
|
| 323 |
+
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
|
| 324 |
+
|
| 325 |
+
``request_context`` must at least contain the ``scheme`` key and its
|
| 326 |
+
value must be a key in ``key_fn_by_scheme`` instance variable.
|
| 327 |
+
"""
|
| 328 |
+
if "strict" in request_context:
|
| 329 |
+
warnings.warn(
|
| 330 |
+
"The 'strict' parameter is no longer needed on Python 3+. "
|
| 331 |
+
"This will raise an error in urllib3 v2.1.0.",
|
| 332 |
+
DeprecationWarning,
|
| 333 |
+
)
|
| 334 |
+
request_context.pop("strict")
|
| 335 |
+
|
| 336 |
+
scheme = request_context["scheme"].lower()
|
| 337 |
+
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
|
| 338 |
+
if not pool_key_constructor:
|
| 339 |
+
raise URLSchemeUnknown(scheme)
|
| 340 |
+
pool_key = pool_key_constructor(request_context)
|
| 341 |
+
|
| 342 |
+
return self.connection_from_pool_key(pool_key, request_context=request_context)
|
| 343 |
+
|
| 344 |
+
def connection_from_pool_key(
|
| 345 |
+
self, pool_key: PoolKey, request_context: dict[str, typing.Any]
|
| 346 |
+
) -> HTTPConnectionPool:
|
| 347 |
+
"""
|
| 348 |
+
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
|
| 349 |
+
|
| 350 |
+
``pool_key`` should be a namedtuple that only contains immutable
|
| 351 |
+
objects. At a minimum it must have the ``scheme``, ``host``, and
|
| 352 |
+
``port`` fields.
|
| 353 |
+
"""
|
| 354 |
+
with self.pools.lock:
|
| 355 |
+
# If the scheme, host, or port doesn't match existing open
|
| 356 |
+
# connections, open a new ConnectionPool.
|
| 357 |
+
pool = self.pools.get(pool_key)
|
| 358 |
+
if pool:
|
| 359 |
+
return pool
|
| 360 |
+
|
| 361 |
+
# Make a fresh ConnectionPool of the desired type
|
| 362 |
+
scheme = request_context["scheme"]
|
| 363 |
+
host = request_context["host"]
|
| 364 |
+
port = request_context["port"]
|
| 365 |
+
pool = self._new_pool(scheme, host, port, request_context=request_context)
|
| 366 |
+
self.pools[pool_key] = pool
|
| 367 |
+
|
| 368 |
+
return pool
|
| 369 |
+
|
| 370 |
+
def connection_from_url(
|
| 371 |
+
self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
|
| 372 |
+
) -> HTTPConnectionPool:
|
| 373 |
+
"""
|
| 374 |
+
Similar to :func:`urllib3.connectionpool.connection_from_url`.
|
| 375 |
+
|
| 376 |
+
If ``pool_kwargs`` is not provided and a new pool needs to be
|
| 377 |
+
constructed, ``self.connection_pool_kw`` is used to initialize
|
| 378 |
+
the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
|
| 379 |
+
is provided, it is used instead. Note that if a new pool does not
|
| 380 |
+
need to be created for the request, the provided ``pool_kwargs`` are
|
| 381 |
+
not used.
|
| 382 |
+
"""
|
| 383 |
+
u = parse_url(url)
|
| 384 |
+
return self.connection_from_host(
|
| 385 |
+
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
|
| 386 |
+
)
|
| 387 |
+
|
| 388 |
+
def _merge_pool_kwargs(
|
| 389 |
+
self, override: dict[str, typing.Any] | None
|
| 390 |
+
) -> dict[str, typing.Any]:
|
| 391 |
+
"""
|
| 392 |
+
Merge a dictionary of override values for self.connection_pool_kw.
|
| 393 |
+
|
| 394 |
+
This does not modify self.connection_pool_kw and returns a new dict.
|
| 395 |
+
Any keys in the override dictionary with a value of ``None`` are
|
| 396 |
+
removed from the merged dictionary.
|
| 397 |
+
"""
|
| 398 |
+
base_pool_kwargs = self.connection_pool_kw.copy()
|
| 399 |
+
if override:
|
| 400 |
+
for key, value in override.items():
|
| 401 |
+
if value is None:
|
| 402 |
+
try:
|
| 403 |
+
del base_pool_kwargs[key]
|
| 404 |
+
except KeyError:
|
| 405 |
+
pass
|
| 406 |
+
else:
|
| 407 |
+
base_pool_kwargs[key] = value
|
| 408 |
+
return base_pool_kwargs
|
| 409 |
+
|
| 410 |
+
def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
|
| 411 |
+
"""
|
| 412 |
+
Indicates if the proxy requires the complete destination URL in the
|
| 413 |
+
request. Normally this is only needed when not using an HTTP CONNECT
|
| 414 |
+
tunnel.
|
| 415 |
+
"""
|
| 416 |
+
if self.proxy is None:
|
| 417 |
+
return False
|
| 418 |
+
|
| 419 |
+
return not connection_requires_http_tunnel(
|
| 420 |
+
self.proxy, self.proxy_config, parsed_url.scheme
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
def urlopen( # type: ignore[override]
|
| 424 |
+
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
| 425 |
+
) -> BaseHTTPResponse:
|
| 426 |
+
"""
|
| 427 |
+
Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
|
| 428 |
+
with custom cross-host redirect logic and only sends the request-uri
|
| 429 |
+
portion of the ``url``.
|
| 430 |
+
|
| 431 |
+
The given ``url`` parameter must be absolute, such that an appropriate
|
| 432 |
+
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
|
| 433 |
+
"""
|
| 434 |
+
u = parse_url(url)
|
| 435 |
+
|
| 436 |
+
if u.scheme is None:
|
| 437 |
+
warnings.warn(
|
| 438 |
+
"URLs without a scheme (ie 'https://') are deprecated and will raise an error "
|
| 439 |
+
"in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs "
|
| 440 |
+
"start with 'https://' or 'http://'. Read more in this issue: "
|
| 441 |
+
"https://github.com/urllib3/urllib3/issues/2920",
|
| 442 |
+
category=DeprecationWarning,
|
| 443 |
+
stacklevel=2,
|
| 444 |
+
)
|
| 445 |
+
|
| 446 |
+
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
|
| 447 |
+
|
| 448 |
+
kw["assert_same_host"] = False
|
| 449 |
+
kw["redirect"] = False
|
| 450 |
+
|
| 451 |
+
if "headers" not in kw:
|
| 452 |
+
kw["headers"] = self.headers
|
| 453 |
+
|
| 454 |
+
if self._proxy_requires_url_absolute_form(u):
|
| 455 |
+
response = conn.urlopen(method, url, **kw)
|
| 456 |
+
else:
|
| 457 |
+
response = conn.urlopen(method, u.request_uri, **kw)
|
| 458 |
+
|
| 459 |
+
redirect_location = redirect and response.get_redirect_location()
|
| 460 |
+
if not redirect_location:
|
| 461 |
+
return response
|
| 462 |
+
|
| 463 |
+
# Support relative URLs for redirecting.
|
| 464 |
+
redirect_location = urljoin(url, redirect_location)
|
| 465 |
+
|
| 466 |
+
if response.status == 303:
|
| 467 |
+
# Change the method according to RFC 9110, Section 15.4.4.
|
| 468 |
+
method = "GET"
|
| 469 |
+
# And lose the body not to transfer anything sensitive.
|
| 470 |
+
kw["body"] = None
|
| 471 |
+
kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
|
| 472 |
+
|
| 473 |
+
retries = kw.get("retries", response.retries)
|
| 474 |
+
if not isinstance(retries, Retry):
|
| 475 |
+
retries = Retry.from_int(retries, redirect=redirect)
|
| 476 |
+
|
| 477 |
+
# Strip headers marked as unsafe to forward to the redirected location.
|
| 478 |
+
# Check remove_headers_on_redirect to avoid a potential network call within
|
| 479 |
+
# conn.is_same_host() which may use socket.gethostbyname() in the future.
|
| 480 |
+
if retries.remove_headers_on_redirect and not conn.is_same_host(
|
| 481 |
+
redirect_location
|
| 482 |
+
):
|
| 483 |
+
new_headers = kw["headers"].copy()
|
| 484 |
+
for header in kw["headers"]:
|
| 485 |
+
if header.lower() in retries.remove_headers_on_redirect:
|
| 486 |
+
new_headers.pop(header, None)
|
| 487 |
+
kw["headers"] = new_headers
|
| 488 |
+
|
| 489 |
+
try:
|
| 490 |
+
retries = retries.increment(method, url, response=response, _pool=conn)
|
| 491 |
+
except MaxRetryError:
|
| 492 |
+
if retries.raise_on_redirect:
|
| 493 |
+
response.drain_conn()
|
| 494 |
+
raise
|
| 495 |
+
return response
|
| 496 |
+
|
| 497 |
+
kw["retries"] = retries
|
| 498 |
+
kw["redirect"] = redirect
|
| 499 |
+
|
| 500 |
+
log.info("Redirecting %s -> %s", url, redirect_location)
|
| 501 |
+
|
| 502 |
+
response.drain_conn()
|
| 503 |
+
return self.urlopen(method, redirect_location, **kw)
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
class ProxyManager(PoolManager):
|
| 507 |
+
"""
|
| 508 |
+
Behaves just like :class:`PoolManager`, but sends all requests through
|
| 509 |
+
the defined proxy, using the CONNECT method for HTTPS URLs.
|
| 510 |
+
|
| 511 |
+
:param proxy_url:
|
| 512 |
+
The URL of the proxy to be used.
|
| 513 |
+
|
| 514 |
+
:param proxy_headers:
|
| 515 |
+
A dictionary containing headers that will be sent to the proxy. In case
|
| 516 |
+
of HTTP they are being sent with each request, while in the
|
| 517 |
+
HTTPS/CONNECT case they are sent only once. Could be used for proxy
|
| 518 |
+
authentication.
|
| 519 |
+
|
| 520 |
+
:param proxy_ssl_context:
|
| 521 |
+
The proxy SSL context is used to establish the TLS connection to the
|
| 522 |
+
proxy when using HTTPS proxies.
|
| 523 |
+
|
| 524 |
+
:param use_forwarding_for_https:
|
| 525 |
+
(Defaults to False) If set to True will forward requests to the HTTPS
|
| 526 |
+
proxy to be made on behalf of the client instead of creating a TLS
|
| 527 |
+
tunnel via the CONNECT method. **Enabling this flag means that request
|
| 528 |
+
and response headers and content will be visible from the HTTPS proxy**
|
| 529 |
+
whereas tunneling keeps request and response headers and content
|
| 530 |
+
private. IP address, target hostname, SNI, and port are always visible
|
| 531 |
+
to an HTTPS proxy even when this flag is disabled.
|
| 532 |
+
|
| 533 |
+
:param proxy_assert_hostname:
|
| 534 |
+
The hostname of the certificate to verify against.
|
| 535 |
+
|
| 536 |
+
:param proxy_assert_fingerprint:
|
| 537 |
+
The fingerprint of the certificate to verify against.
|
| 538 |
+
|
| 539 |
+
Example:
|
| 540 |
+
|
| 541 |
+
.. code-block:: python
|
| 542 |
+
|
| 543 |
+
import urllib3
|
| 544 |
+
|
| 545 |
+
proxy = urllib3.ProxyManager("https://localhost:3128/")
|
| 546 |
+
|
| 547 |
+
resp1 = proxy.request("GET", "https://google.com/")
|
| 548 |
+
resp2 = proxy.request("GET", "https://httpbin.org/")
|
| 549 |
+
|
| 550 |
+
print(len(proxy.pools))
|
| 551 |
+
# 1
|
| 552 |
+
|
| 553 |
+
resp3 = proxy.request("GET", "https://httpbin.org/")
|
| 554 |
+
resp4 = proxy.request("GET", "https://twitter.com/")
|
| 555 |
+
|
| 556 |
+
print(len(proxy.pools))
|
| 557 |
+
# 3
|
| 558 |
+
|
| 559 |
+
"""
|
| 560 |
+
|
| 561 |
+
def __init__(
|
| 562 |
+
self,
|
| 563 |
+
proxy_url: str,
|
| 564 |
+
num_pools: int = 10,
|
| 565 |
+
headers: typing.Mapping[str, str] | None = None,
|
| 566 |
+
proxy_headers: typing.Mapping[str, str] | None = None,
|
| 567 |
+
proxy_ssl_context: ssl.SSLContext | None = None,
|
| 568 |
+
use_forwarding_for_https: bool = False,
|
| 569 |
+
proxy_assert_hostname: None | str | typing.Literal[False] = None,
|
| 570 |
+
proxy_assert_fingerprint: str | None = None,
|
| 571 |
+
**connection_pool_kw: typing.Any,
|
| 572 |
+
) -> None:
|
| 573 |
+
if isinstance(proxy_url, HTTPConnectionPool):
|
| 574 |
+
str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
|
| 575 |
+
else:
|
| 576 |
+
str_proxy_url = proxy_url
|
| 577 |
+
proxy = parse_url(str_proxy_url)
|
| 578 |
+
|
| 579 |
+
if proxy.scheme not in ("http", "https"):
|
| 580 |
+
raise ProxySchemeUnknown(proxy.scheme)
|
| 581 |
+
|
| 582 |
+
if not proxy.port:
|
| 583 |
+
port = port_by_scheme.get(proxy.scheme, 80)
|
| 584 |
+
proxy = proxy._replace(port=port)
|
| 585 |
+
|
| 586 |
+
self.proxy = proxy
|
| 587 |
+
self.proxy_headers = proxy_headers or {}
|
| 588 |
+
self.proxy_ssl_context = proxy_ssl_context
|
| 589 |
+
self.proxy_config = ProxyConfig(
|
| 590 |
+
proxy_ssl_context,
|
| 591 |
+
use_forwarding_for_https,
|
| 592 |
+
proxy_assert_hostname,
|
| 593 |
+
proxy_assert_fingerprint,
|
| 594 |
+
)
|
| 595 |
+
|
| 596 |
+
connection_pool_kw["_proxy"] = self.proxy
|
| 597 |
+
connection_pool_kw["_proxy_headers"] = self.proxy_headers
|
| 598 |
+
connection_pool_kw["_proxy_config"] = self.proxy_config
|
| 599 |
+
|
| 600 |
+
super().__init__(num_pools, headers, **connection_pool_kw)
|
| 601 |
+
|
| 602 |
+
def connection_from_host(
|
| 603 |
+
self,
|
| 604 |
+
host: str | None,
|
| 605 |
+
port: int | None = None,
|
| 606 |
+
scheme: str | None = "http",
|
| 607 |
+
pool_kwargs: dict[str, typing.Any] | None = None,
|
| 608 |
+
) -> HTTPConnectionPool:
|
| 609 |
+
if scheme == "https":
|
| 610 |
+
return super().connection_from_host(
|
| 611 |
+
host, port, scheme, pool_kwargs=pool_kwargs
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
return super().connection_from_host(
|
| 615 |
+
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
|
| 616 |
+
)
|
| 617 |
+
|
| 618 |
+
def _set_proxy_headers(
|
| 619 |
+
self, url: str, headers: typing.Mapping[str, str] | None = None
|
| 620 |
+
) -> typing.Mapping[str, str]:
|
| 621 |
+
"""
|
| 622 |
+
Sets headers needed by proxies: specifically, the Accept and Host
|
| 623 |
+
headers. Only sets headers not provided by the user.
|
| 624 |
+
"""
|
| 625 |
+
headers_ = {"Accept": "*/*"}
|
| 626 |
+
|
| 627 |
+
netloc = parse_url(url).netloc
|
| 628 |
+
if netloc:
|
| 629 |
+
headers_["Host"] = netloc
|
| 630 |
+
|
| 631 |
+
if headers:
|
| 632 |
+
headers_.update(headers)
|
| 633 |
+
return headers_
|
| 634 |
+
|
| 635 |
+
def urlopen( # type: ignore[override]
|
| 636 |
+
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
| 637 |
+
) -> BaseHTTPResponse:
|
| 638 |
+
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
|
| 639 |
+
u = parse_url(url)
|
| 640 |
+
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
|
| 641 |
+
# For connections using HTTP CONNECT, httplib sets the necessary
|
| 642 |
+
# headers on the CONNECT to the proxy. If we're not using CONNECT,
|
| 643 |
+
# we'll definitely need to set 'Host' at the very least.
|
| 644 |
+
headers = kw.get("headers", self.headers)
|
| 645 |
+
kw["headers"] = self._set_proxy_headers(url, headers)
|
| 646 |
+
|
| 647 |
+
return super().urlopen(method, url, redirect=redirect, **kw)
|
| 648 |
+
|
| 649 |
+
|
| 650 |
+
def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
|
| 651 |
+
return ProxyManager(proxy_url=url, **kw)
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/py.typed
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Instruct type checkers to look for inline type annotations in this package.
|
| 2 |
+
# See PEP 561.
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/response.py
ADDED
|
@@ -0,0 +1,1480 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import collections
|
| 4 |
+
import io
|
| 5 |
+
import json as _json
|
| 6 |
+
import logging
|
| 7 |
+
import socket
|
| 8 |
+
import sys
|
| 9 |
+
import typing
|
| 10 |
+
import warnings
|
| 11 |
+
import zlib
|
| 12 |
+
from contextlib import contextmanager
|
| 13 |
+
from http.client import HTTPMessage as _HttplibHTTPMessage
|
| 14 |
+
from http.client import HTTPResponse as _HttplibHTTPResponse
|
| 15 |
+
from socket import timeout as SocketTimeout
|
| 16 |
+
|
| 17 |
+
if typing.TYPE_CHECKING:
|
| 18 |
+
from ._base_connection import BaseHTTPConnection
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
try:
|
| 22 |
+
import brotlicffi as brotli # type: ignore[import-not-found]
|
| 23 |
+
except ImportError:
|
| 24 |
+
import brotli # type: ignore[import-not-found]
|
| 25 |
+
except ImportError:
|
| 26 |
+
brotli = None
|
| 27 |
+
|
| 28 |
+
from . import util
|
| 29 |
+
from ._base_connection import _TYPE_BODY
|
| 30 |
+
from ._collections import HTTPHeaderDict
|
| 31 |
+
from .connection import BaseSSLError, HTTPConnection, HTTPException
|
| 32 |
+
from .exceptions import (
|
| 33 |
+
BodyNotHttplibCompatible,
|
| 34 |
+
DecodeError,
|
| 35 |
+
DependencyWarning,
|
| 36 |
+
HTTPError,
|
| 37 |
+
IncompleteRead,
|
| 38 |
+
InvalidChunkLength,
|
| 39 |
+
InvalidHeader,
|
| 40 |
+
ProtocolError,
|
| 41 |
+
ReadTimeoutError,
|
| 42 |
+
ResponseNotChunked,
|
| 43 |
+
SSLError,
|
| 44 |
+
)
|
| 45 |
+
from .util.response import is_fp_closed, is_response_to_head
|
| 46 |
+
from .util.retry import Retry
|
| 47 |
+
|
| 48 |
+
if typing.TYPE_CHECKING:
|
| 49 |
+
from .connectionpool import HTTPConnectionPool
|
| 50 |
+
|
| 51 |
+
log = logging.getLogger(__name__)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class ContentDecoder:
|
| 55 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 56 |
+
raise NotImplementedError()
|
| 57 |
+
|
| 58 |
+
@property
|
| 59 |
+
def has_unconsumed_tail(self) -> bool:
|
| 60 |
+
raise NotImplementedError()
|
| 61 |
+
|
| 62 |
+
def flush(self) -> bytes:
|
| 63 |
+
raise NotImplementedError()
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class DeflateDecoder(ContentDecoder):
|
| 67 |
+
def __init__(self) -> None:
|
| 68 |
+
self._first_try = True
|
| 69 |
+
self._first_try_data = b""
|
| 70 |
+
self._unfed_data = b""
|
| 71 |
+
self._obj = zlib.decompressobj()
|
| 72 |
+
|
| 73 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 74 |
+
data = self._unfed_data + data
|
| 75 |
+
self._unfed_data = b""
|
| 76 |
+
if not data and not self._obj.unconsumed_tail:
|
| 77 |
+
return data
|
| 78 |
+
original_max_length = max_length
|
| 79 |
+
if original_max_length < 0:
|
| 80 |
+
max_length = 0
|
| 81 |
+
elif original_max_length == 0:
|
| 82 |
+
# We should not pass 0 to the zlib decompressor because 0 is
|
| 83 |
+
# the default value that will make zlib decompress without a
|
| 84 |
+
# length limit.
|
| 85 |
+
# Data should be stored for subsequent calls.
|
| 86 |
+
self._unfed_data = data
|
| 87 |
+
return b""
|
| 88 |
+
|
| 89 |
+
# Subsequent calls always reuse `self._obj`. zlib requires
|
| 90 |
+
# passing the unconsumed tail if decompression is to continue.
|
| 91 |
+
if not self._first_try:
|
| 92 |
+
return self._obj.decompress(
|
| 93 |
+
self._obj.unconsumed_tail + data, max_length=max_length
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
# First call tries with RFC 1950 ZLIB format.
|
| 97 |
+
self._first_try_data += data
|
| 98 |
+
try:
|
| 99 |
+
decompressed = self._obj.decompress(data, max_length=max_length)
|
| 100 |
+
if decompressed:
|
| 101 |
+
self._first_try = False
|
| 102 |
+
self._first_try_data = b""
|
| 103 |
+
return decompressed
|
| 104 |
+
# On failure, it falls back to RFC 1951 DEFLATE format.
|
| 105 |
+
except zlib.error:
|
| 106 |
+
self._first_try = False
|
| 107 |
+
self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
|
| 108 |
+
try:
|
| 109 |
+
return self.decompress(
|
| 110 |
+
self._first_try_data, max_length=original_max_length
|
| 111 |
+
)
|
| 112 |
+
finally:
|
| 113 |
+
self._first_try_data = b""
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def has_unconsumed_tail(self) -> bool:
|
| 117 |
+
return bool(self._unfed_data) or (
|
| 118 |
+
bool(self._obj.unconsumed_tail) and not self._first_try
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
def flush(self) -> bytes:
|
| 122 |
+
return self._obj.flush()
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
class GzipDecoderState:
|
| 126 |
+
FIRST_MEMBER = 0
|
| 127 |
+
OTHER_MEMBERS = 1
|
| 128 |
+
SWALLOW_DATA = 2
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
class GzipDecoder(ContentDecoder):
|
| 132 |
+
def __init__(self) -> None:
|
| 133 |
+
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
| 134 |
+
self._state = GzipDecoderState.FIRST_MEMBER
|
| 135 |
+
self._unconsumed_tail = b""
|
| 136 |
+
|
| 137 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 138 |
+
ret = bytearray()
|
| 139 |
+
if self._state == GzipDecoderState.SWALLOW_DATA:
|
| 140 |
+
return bytes(ret)
|
| 141 |
+
|
| 142 |
+
if max_length == 0:
|
| 143 |
+
# We should not pass 0 to the zlib decompressor because 0 is
|
| 144 |
+
# the default value that will make zlib decompress without a
|
| 145 |
+
# length limit.
|
| 146 |
+
# Data should be stored for subsequent calls.
|
| 147 |
+
self._unconsumed_tail += data
|
| 148 |
+
return b""
|
| 149 |
+
|
| 150 |
+
# zlib requires passing the unconsumed tail to the subsequent
|
| 151 |
+
# call if decompression is to continue.
|
| 152 |
+
data = self._unconsumed_tail + data
|
| 153 |
+
if not data and self._obj.eof:
|
| 154 |
+
return bytes(ret)
|
| 155 |
+
|
| 156 |
+
while True:
|
| 157 |
+
try:
|
| 158 |
+
ret += self._obj.decompress(
|
| 159 |
+
data, max_length=max(max_length - len(ret), 0)
|
| 160 |
+
)
|
| 161 |
+
except zlib.error:
|
| 162 |
+
previous_state = self._state
|
| 163 |
+
# Ignore data after the first error
|
| 164 |
+
self._state = GzipDecoderState.SWALLOW_DATA
|
| 165 |
+
self._unconsumed_tail = b""
|
| 166 |
+
if previous_state == GzipDecoderState.OTHER_MEMBERS:
|
| 167 |
+
# Allow trailing garbage acceptable in other gzip clients
|
| 168 |
+
return bytes(ret)
|
| 169 |
+
raise
|
| 170 |
+
|
| 171 |
+
self._unconsumed_tail = data = (
|
| 172 |
+
self._obj.unconsumed_tail or self._obj.unused_data
|
| 173 |
+
)
|
| 174 |
+
if max_length > 0 and len(ret) >= max_length:
|
| 175 |
+
break
|
| 176 |
+
|
| 177 |
+
if not data:
|
| 178 |
+
return bytes(ret)
|
| 179 |
+
# When the end of a gzip member is reached, a new decompressor
|
| 180 |
+
# must be created for unused (possibly future) data.
|
| 181 |
+
if self._obj.eof:
|
| 182 |
+
self._state = GzipDecoderState.OTHER_MEMBERS
|
| 183 |
+
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
| 184 |
+
|
| 185 |
+
return bytes(ret)
|
| 186 |
+
|
| 187 |
+
@property
|
| 188 |
+
def has_unconsumed_tail(self) -> bool:
|
| 189 |
+
return bool(self._unconsumed_tail)
|
| 190 |
+
|
| 191 |
+
def flush(self) -> bytes:
|
| 192 |
+
return self._obj.flush()
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
if brotli is not None:
|
| 196 |
+
|
| 197 |
+
class BrotliDecoder(ContentDecoder):
|
| 198 |
+
# Supports both 'brotlipy' and 'Brotli' packages
|
| 199 |
+
# since they share an import name. The top branches
|
| 200 |
+
# are for 'brotlipy' and bottom branches for 'Brotli'
|
| 201 |
+
def __init__(self) -> None:
|
| 202 |
+
self._obj = brotli.Decompressor()
|
| 203 |
+
if hasattr(self._obj, "decompress"):
|
| 204 |
+
setattr(self, "_decompress", self._obj.decompress)
|
| 205 |
+
else:
|
| 206 |
+
setattr(self, "_decompress", self._obj.process)
|
| 207 |
+
|
| 208 |
+
# Requires Brotli >= 1.2.0 for `output_buffer_limit`.
|
| 209 |
+
def _decompress(self, data: bytes, output_buffer_limit: int = -1) -> bytes:
|
| 210 |
+
raise NotImplementedError()
|
| 211 |
+
|
| 212 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 213 |
+
try:
|
| 214 |
+
if max_length > 0:
|
| 215 |
+
return self._decompress(data, output_buffer_limit=max_length)
|
| 216 |
+
else:
|
| 217 |
+
return self._decompress(data)
|
| 218 |
+
except TypeError:
|
| 219 |
+
# Fallback for Brotli/brotlicffi/brotlipy versions without
|
| 220 |
+
# the `output_buffer_limit` parameter.
|
| 221 |
+
warnings.warn(
|
| 222 |
+
"Brotli >= 1.2.0 is required to prevent decompression bombs.",
|
| 223 |
+
DependencyWarning,
|
| 224 |
+
)
|
| 225 |
+
return self._decompress(data)
|
| 226 |
+
|
| 227 |
+
@property
|
| 228 |
+
def has_unconsumed_tail(self) -> bool:
|
| 229 |
+
try:
|
| 230 |
+
return not self._obj.can_accept_more_data()
|
| 231 |
+
except AttributeError:
|
| 232 |
+
return False
|
| 233 |
+
|
| 234 |
+
def flush(self) -> bytes:
|
| 235 |
+
if hasattr(self._obj, "flush"):
|
| 236 |
+
return self._obj.flush() # type: ignore[no-any-return]
|
| 237 |
+
return b""
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
try:
|
| 241 |
+
if sys.version_info >= (3, 14):
|
| 242 |
+
from compression import zstd
|
| 243 |
+
else:
|
| 244 |
+
from backports import zstd
|
| 245 |
+
except ImportError:
|
| 246 |
+
HAS_ZSTD = False
|
| 247 |
+
else:
|
| 248 |
+
HAS_ZSTD = True
|
| 249 |
+
|
| 250 |
+
class ZstdDecoder(ContentDecoder):
|
| 251 |
+
def __init__(self) -> None:
|
| 252 |
+
self._obj = zstd.ZstdDecompressor()
|
| 253 |
+
|
| 254 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 255 |
+
if not data and not self.has_unconsumed_tail:
|
| 256 |
+
return b""
|
| 257 |
+
if self._obj.eof:
|
| 258 |
+
data = self._obj.unused_data + data
|
| 259 |
+
self._obj = zstd.ZstdDecompressor()
|
| 260 |
+
part = self._obj.decompress(data, max_length=max_length)
|
| 261 |
+
length = len(part)
|
| 262 |
+
data_parts = [part]
|
| 263 |
+
# Every loop iteration is supposed to read data from a separate frame.
|
| 264 |
+
# The loop breaks when:
|
| 265 |
+
# - enough data is read;
|
| 266 |
+
# - no more unused data is available;
|
| 267 |
+
# - end of the last read frame has not been reached (i.e.,
|
| 268 |
+
# more data has to be fed).
|
| 269 |
+
while (
|
| 270 |
+
self._obj.eof
|
| 271 |
+
and self._obj.unused_data
|
| 272 |
+
and (max_length < 0 or length < max_length)
|
| 273 |
+
):
|
| 274 |
+
unused_data = self._obj.unused_data
|
| 275 |
+
if not self._obj.needs_input:
|
| 276 |
+
self._obj = zstd.ZstdDecompressor()
|
| 277 |
+
part = self._obj.decompress(
|
| 278 |
+
unused_data,
|
| 279 |
+
max_length=(max_length - length) if max_length > 0 else -1,
|
| 280 |
+
)
|
| 281 |
+
if part_length := len(part):
|
| 282 |
+
data_parts.append(part)
|
| 283 |
+
length += part_length
|
| 284 |
+
elif self._obj.needs_input:
|
| 285 |
+
break
|
| 286 |
+
return b"".join(data_parts)
|
| 287 |
+
|
| 288 |
+
@property
|
| 289 |
+
def has_unconsumed_tail(self) -> bool:
|
| 290 |
+
return not (self._obj.needs_input or self._obj.eof) or bool(
|
| 291 |
+
self._obj.unused_data
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
def flush(self) -> bytes:
|
| 295 |
+
if not self._obj.eof:
|
| 296 |
+
raise DecodeError("Zstandard data is incomplete")
|
| 297 |
+
return b""
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
class MultiDecoder(ContentDecoder):
|
| 301 |
+
"""
|
| 302 |
+
From RFC7231:
|
| 303 |
+
If one or more encodings have been applied to a representation, the
|
| 304 |
+
sender that applied the encodings MUST generate a Content-Encoding
|
| 305 |
+
header field that lists the content codings in the order in which
|
| 306 |
+
they were applied.
|
| 307 |
+
"""
|
| 308 |
+
|
| 309 |
+
# Maximum allowed number of chained HTTP encodings in the
|
| 310 |
+
# Content-Encoding header.
|
| 311 |
+
max_decode_links = 5
|
| 312 |
+
|
| 313 |
+
def __init__(self, modes: str) -> None:
|
| 314 |
+
encodings = [m.strip() for m in modes.split(",")]
|
| 315 |
+
if len(encodings) > self.max_decode_links:
|
| 316 |
+
raise DecodeError(
|
| 317 |
+
"Too many content encodings in the chain: "
|
| 318 |
+
f"{len(encodings)} > {self.max_decode_links}"
|
| 319 |
+
)
|
| 320 |
+
self._decoders = [_get_decoder(e) for e in encodings]
|
| 321 |
+
|
| 322 |
+
def flush(self) -> bytes:
|
| 323 |
+
return self._decoders[0].flush()
|
| 324 |
+
|
| 325 |
+
def decompress(self, data: bytes, max_length: int = -1) -> bytes:
|
| 326 |
+
if max_length <= 0:
|
| 327 |
+
for d in reversed(self._decoders):
|
| 328 |
+
data = d.decompress(data)
|
| 329 |
+
return data
|
| 330 |
+
|
| 331 |
+
ret = bytearray()
|
| 332 |
+
# Every while loop iteration goes through all decoders once.
|
| 333 |
+
# It exits when enough data is read or no more data can be read.
|
| 334 |
+
# It is possible that the while loop iteration does not produce
|
| 335 |
+
# any data because we retrieve up to `max_length` from every
|
| 336 |
+
# decoder, and the amount of bytes may be insufficient for the
|
| 337 |
+
# next decoder to produce enough/any output.
|
| 338 |
+
while True:
|
| 339 |
+
any_data = False
|
| 340 |
+
for d in reversed(self._decoders):
|
| 341 |
+
data = d.decompress(data, max_length=max_length - len(ret))
|
| 342 |
+
if data:
|
| 343 |
+
any_data = True
|
| 344 |
+
# We should not break when no data is returned because
|
| 345 |
+
# next decoders may produce data even with empty input.
|
| 346 |
+
ret += data
|
| 347 |
+
if not any_data or len(ret) >= max_length:
|
| 348 |
+
return bytes(ret)
|
| 349 |
+
data = b""
|
| 350 |
+
|
| 351 |
+
@property
|
| 352 |
+
def has_unconsumed_tail(self) -> bool:
|
| 353 |
+
return any(d.has_unconsumed_tail for d in self._decoders)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def _get_decoder(mode: str) -> ContentDecoder:
|
| 357 |
+
if "," in mode:
|
| 358 |
+
return MultiDecoder(mode)
|
| 359 |
+
|
| 360 |
+
# According to RFC 9110 section 8.4.1.3, recipients should
|
| 361 |
+
# consider x-gzip equivalent to gzip
|
| 362 |
+
if mode in ("gzip", "x-gzip"):
|
| 363 |
+
return GzipDecoder()
|
| 364 |
+
|
| 365 |
+
if brotli is not None and mode == "br":
|
| 366 |
+
return BrotliDecoder()
|
| 367 |
+
|
| 368 |
+
if HAS_ZSTD and mode == "zstd":
|
| 369 |
+
return ZstdDecoder()
|
| 370 |
+
|
| 371 |
+
return DeflateDecoder()
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
class BytesQueueBuffer:
|
| 375 |
+
"""Memory-efficient bytes buffer
|
| 376 |
+
|
| 377 |
+
To return decoded data in read() and still follow the BufferedIOBase API, we need a
|
| 378 |
+
buffer to always return the correct amount of bytes.
|
| 379 |
+
|
| 380 |
+
This buffer should be filled using calls to put()
|
| 381 |
+
|
| 382 |
+
Our maximum memory usage is determined by the sum of the size of:
|
| 383 |
+
|
| 384 |
+
* self.buffer, which contains the full data
|
| 385 |
+
* the largest chunk that we will copy in get()
|
| 386 |
+
"""
|
| 387 |
+
|
| 388 |
+
def __init__(self) -> None:
|
| 389 |
+
self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque()
|
| 390 |
+
self._size: int = 0
|
| 391 |
+
|
| 392 |
+
def __len__(self) -> int:
|
| 393 |
+
return self._size
|
| 394 |
+
|
| 395 |
+
def put(self, data: bytes) -> None:
|
| 396 |
+
self.buffer.append(data)
|
| 397 |
+
self._size += len(data)
|
| 398 |
+
|
| 399 |
+
def get(self, n: int) -> bytes:
|
| 400 |
+
if n == 0:
|
| 401 |
+
return b""
|
| 402 |
+
elif not self.buffer:
|
| 403 |
+
raise RuntimeError("buffer is empty")
|
| 404 |
+
elif n < 0:
|
| 405 |
+
raise ValueError("n should be > 0")
|
| 406 |
+
|
| 407 |
+
if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes):
|
| 408 |
+
self._size -= n
|
| 409 |
+
return self.buffer.popleft()
|
| 410 |
+
|
| 411 |
+
fetched = 0
|
| 412 |
+
ret = io.BytesIO()
|
| 413 |
+
while fetched < n:
|
| 414 |
+
remaining = n - fetched
|
| 415 |
+
chunk = self.buffer.popleft()
|
| 416 |
+
chunk_length = len(chunk)
|
| 417 |
+
if remaining < chunk_length:
|
| 418 |
+
chunk = memoryview(chunk)
|
| 419 |
+
left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
|
| 420 |
+
ret.write(left_chunk)
|
| 421 |
+
self.buffer.appendleft(right_chunk)
|
| 422 |
+
self._size -= remaining
|
| 423 |
+
break
|
| 424 |
+
else:
|
| 425 |
+
ret.write(chunk)
|
| 426 |
+
self._size -= chunk_length
|
| 427 |
+
fetched += chunk_length
|
| 428 |
+
|
| 429 |
+
if not self.buffer:
|
| 430 |
+
break
|
| 431 |
+
|
| 432 |
+
return ret.getvalue()
|
| 433 |
+
|
| 434 |
+
def get_all(self) -> bytes:
|
| 435 |
+
buffer = self.buffer
|
| 436 |
+
if not buffer:
|
| 437 |
+
assert self._size == 0
|
| 438 |
+
return b""
|
| 439 |
+
if len(buffer) == 1:
|
| 440 |
+
result = buffer.pop()
|
| 441 |
+
if isinstance(result, memoryview):
|
| 442 |
+
result = result.tobytes()
|
| 443 |
+
else:
|
| 444 |
+
ret = io.BytesIO()
|
| 445 |
+
ret.writelines(buffer.popleft() for _ in range(len(buffer)))
|
| 446 |
+
result = ret.getvalue()
|
| 447 |
+
self._size = 0
|
| 448 |
+
return result
|
| 449 |
+
|
| 450 |
+
|
| 451 |
+
class BaseHTTPResponse(io.IOBase):
|
| 452 |
+
CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
|
| 453 |
+
if brotli is not None:
|
| 454 |
+
CONTENT_DECODERS += ["br"]
|
| 455 |
+
if HAS_ZSTD:
|
| 456 |
+
CONTENT_DECODERS += ["zstd"]
|
| 457 |
+
REDIRECT_STATUSES = [301, 302, 303, 307, 308]
|
| 458 |
+
|
| 459 |
+
DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
|
| 460 |
+
if brotli is not None:
|
| 461 |
+
DECODER_ERROR_CLASSES += (brotli.error,)
|
| 462 |
+
|
| 463 |
+
if HAS_ZSTD:
|
| 464 |
+
DECODER_ERROR_CLASSES += (zstd.ZstdError,)
|
| 465 |
+
|
| 466 |
+
def __init__(
|
| 467 |
+
self,
|
| 468 |
+
*,
|
| 469 |
+
headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
|
| 470 |
+
status: int,
|
| 471 |
+
version: int,
|
| 472 |
+
version_string: str,
|
| 473 |
+
reason: str | None,
|
| 474 |
+
decode_content: bool,
|
| 475 |
+
request_url: str | None,
|
| 476 |
+
retries: Retry | None = None,
|
| 477 |
+
) -> None:
|
| 478 |
+
if isinstance(headers, HTTPHeaderDict):
|
| 479 |
+
self.headers = headers
|
| 480 |
+
else:
|
| 481 |
+
self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type]
|
| 482 |
+
self.status = status
|
| 483 |
+
self.version = version
|
| 484 |
+
self.version_string = version_string
|
| 485 |
+
self.reason = reason
|
| 486 |
+
self.decode_content = decode_content
|
| 487 |
+
self._has_decoded_content = False
|
| 488 |
+
self._request_url: str | None = request_url
|
| 489 |
+
self.retries = retries
|
| 490 |
+
|
| 491 |
+
self.chunked = False
|
| 492 |
+
tr_enc = self.headers.get("transfer-encoding", "").lower()
|
| 493 |
+
# Don't incur the penalty of creating a list and then discarding it
|
| 494 |
+
encodings = (enc.strip() for enc in tr_enc.split(","))
|
| 495 |
+
if "chunked" in encodings:
|
| 496 |
+
self.chunked = True
|
| 497 |
+
|
| 498 |
+
self._decoder: ContentDecoder | None = None
|
| 499 |
+
self.length_remaining: int | None
|
| 500 |
+
|
| 501 |
+
def get_redirect_location(self) -> str | None | typing.Literal[False]:
|
| 502 |
+
"""
|
| 503 |
+
Should we redirect and where to?
|
| 504 |
+
|
| 505 |
+
:returns: Truthy redirect location string if we got a redirect status
|
| 506 |
+
code and valid location. ``None`` if redirect status and no
|
| 507 |
+
location. ``False`` if not a redirect status code.
|
| 508 |
+
"""
|
| 509 |
+
if self.status in self.REDIRECT_STATUSES:
|
| 510 |
+
return self.headers.get("location")
|
| 511 |
+
return False
|
| 512 |
+
|
| 513 |
+
@property
|
| 514 |
+
def data(self) -> bytes:
|
| 515 |
+
raise NotImplementedError()
|
| 516 |
+
|
| 517 |
+
def json(self) -> typing.Any:
|
| 518 |
+
"""
|
| 519 |
+
Deserializes the body of the HTTP response as a Python object.
|
| 520 |
+
|
| 521 |
+
The body of the HTTP response must be encoded using UTF-8, as per
|
| 522 |
+
`RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
|
| 523 |
+
|
| 524 |
+
To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
|
| 525 |
+
your custom decoder instead.
|
| 526 |
+
|
| 527 |
+
If the body of the HTTP response is not decodable to UTF-8, a
|
| 528 |
+
`UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
|
| 529 |
+
valid JSON document, a `json.JSONDecodeError` will be raised.
|
| 530 |
+
|
| 531 |
+
Read more :ref:`here <json_content>`.
|
| 532 |
+
|
| 533 |
+
:returns: The body of the HTTP response as a Python object.
|
| 534 |
+
"""
|
| 535 |
+
data = self.data.decode("utf-8")
|
| 536 |
+
return _json.loads(data)
|
| 537 |
+
|
| 538 |
+
@property
|
| 539 |
+
def url(self) -> str | None:
|
| 540 |
+
raise NotImplementedError()
|
| 541 |
+
|
| 542 |
+
@url.setter
|
| 543 |
+
def url(self, url: str | None) -> None:
|
| 544 |
+
raise NotImplementedError()
|
| 545 |
+
|
| 546 |
+
@property
|
| 547 |
+
def connection(self) -> BaseHTTPConnection | None:
|
| 548 |
+
raise NotImplementedError()
|
| 549 |
+
|
| 550 |
+
@property
|
| 551 |
+
def retries(self) -> Retry | None:
|
| 552 |
+
return self._retries
|
| 553 |
+
|
| 554 |
+
@retries.setter
|
| 555 |
+
def retries(self, retries: Retry | None) -> None:
|
| 556 |
+
# Override the request_url if retries has a redirect location.
|
| 557 |
+
if retries is not None and retries.history:
|
| 558 |
+
self.url = retries.history[-1].redirect_location
|
| 559 |
+
self._retries = retries
|
| 560 |
+
|
| 561 |
+
def stream(
|
| 562 |
+
self, amt: int | None = 2**16, decode_content: bool | None = None
|
| 563 |
+
) -> typing.Iterator[bytes]:
|
| 564 |
+
raise NotImplementedError()
|
| 565 |
+
|
| 566 |
+
def read(
|
| 567 |
+
self,
|
| 568 |
+
amt: int | None = None,
|
| 569 |
+
decode_content: bool | None = None,
|
| 570 |
+
cache_content: bool = False,
|
| 571 |
+
) -> bytes:
|
| 572 |
+
raise NotImplementedError()
|
| 573 |
+
|
| 574 |
+
def read1(
|
| 575 |
+
self,
|
| 576 |
+
amt: int | None = None,
|
| 577 |
+
decode_content: bool | None = None,
|
| 578 |
+
) -> bytes:
|
| 579 |
+
raise NotImplementedError()
|
| 580 |
+
|
| 581 |
+
def read_chunked(
|
| 582 |
+
self,
|
| 583 |
+
amt: int | None = None,
|
| 584 |
+
decode_content: bool | None = None,
|
| 585 |
+
) -> typing.Iterator[bytes]:
|
| 586 |
+
raise NotImplementedError()
|
| 587 |
+
|
| 588 |
+
def release_conn(self) -> None:
|
| 589 |
+
raise NotImplementedError()
|
| 590 |
+
|
| 591 |
+
def drain_conn(self) -> None:
|
| 592 |
+
raise NotImplementedError()
|
| 593 |
+
|
| 594 |
+
def shutdown(self) -> None:
|
| 595 |
+
raise NotImplementedError()
|
| 596 |
+
|
| 597 |
+
def close(self) -> None:
|
| 598 |
+
raise NotImplementedError()
|
| 599 |
+
|
| 600 |
+
def _init_decoder(self) -> None:
|
| 601 |
+
"""
|
| 602 |
+
Set-up the _decoder attribute if necessary.
|
| 603 |
+
"""
|
| 604 |
+
# Note: content-encoding value should be case-insensitive, per RFC 7230
|
| 605 |
+
# Section 3.2
|
| 606 |
+
content_encoding = self.headers.get("content-encoding", "").lower()
|
| 607 |
+
if self._decoder is None:
|
| 608 |
+
if content_encoding in self.CONTENT_DECODERS:
|
| 609 |
+
self._decoder = _get_decoder(content_encoding)
|
| 610 |
+
elif "," in content_encoding:
|
| 611 |
+
encodings = [
|
| 612 |
+
e.strip()
|
| 613 |
+
for e in content_encoding.split(",")
|
| 614 |
+
if e.strip() in self.CONTENT_DECODERS
|
| 615 |
+
]
|
| 616 |
+
if encodings:
|
| 617 |
+
self._decoder = _get_decoder(content_encoding)
|
| 618 |
+
|
| 619 |
+
def _decode(
|
| 620 |
+
self,
|
| 621 |
+
data: bytes,
|
| 622 |
+
decode_content: bool | None,
|
| 623 |
+
flush_decoder: bool,
|
| 624 |
+
max_length: int | None = None,
|
| 625 |
+
) -> bytes:
|
| 626 |
+
"""
|
| 627 |
+
Decode the data passed in and potentially flush the decoder.
|
| 628 |
+
"""
|
| 629 |
+
if not decode_content:
|
| 630 |
+
if self._has_decoded_content:
|
| 631 |
+
raise RuntimeError(
|
| 632 |
+
"Calling read(decode_content=False) is not supported after "
|
| 633 |
+
"read(decode_content=True) was called."
|
| 634 |
+
)
|
| 635 |
+
return data
|
| 636 |
+
|
| 637 |
+
if max_length is None or flush_decoder:
|
| 638 |
+
max_length = -1
|
| 639 |
+
|
| 640 |
+
try:
|
| 641 |
+
if self._decoder:
|
| 642 |
+
data = self._decoder.decompress(data, max_length=max_length)
|
| 643 |
+
self._has_decoded_content = True
|
| 644 |
+
except self.DECODER_ERROR_CLASSES as e:
|
| 645 |
+
content_encoding = self.headers.get("content-encoding", "").lower()
|
| 646 |
+
raise DecodeError(
|
| 647 |
+
"Received response with content-encoding: %s, but "
|
| 648 |
+
"failed to decode it." % content_encoding,
|
| 649 |
+
e,
|
| 650 |
+
) from e
|
| 651 |
+
if flush_decoder:
|
| 652 |
+
data += self._flush_decoder()
|
| 653 |
+
|
| 654 |
+
return data
|
| 655 |
+
|
| 656 |
+
def _flush_decoder(self) -> bytes:
|
| 657 |
+
"""
|
| 658 |
+
Flushes the decoder. Should only be called if the decoder is actually
|
| 659 |
+
being used.
|
| 660 |
+
"""
|
| 661 |
+
if self._decoder:
|
| 662 |
+
return self._decoder.decompress(b"") + self._decoder.flush()
|
| 663 |
+
return b""
|
| 664 |
+
|
| 665 |
+
# Compatibility methods for `io` module
|
| 666 |
+
def readinto(self, b: bytearray) -> int:
|
| 667 |
+
temp = self.read(len(b))
|
| 668 |
+
if len(temp) == 0:
|
| 669 |
+
return 0
|
| 670 |
+
else:
|
| 671 |
+
b[: len(temp)] = temp
|
| 672 |
+
return len(temp)
|
| 673 |
+
|
| 674 |
+
# Methods used by dependent libraries
|
| 675 |
+
def getheaders(self) -> HTTPHeaderDict:
|
| 676 |
+
return self.headers
|
| 677 |
+
|
| 678 |
+
def getheader(self, name: str, default: str | None = None) -> str | None:
|
| 679 |
+
return self.headers.get(name, default)
|
| 680 |
+
|
| 681 |
+
# Compatibility method for http.cookiejar
|
| 682 |
+
def info(self) -> HTTPHeaderDict:
|
| 683 |
+
return self.headers
|
| 684 |
+
|
| 685 |
+
def geturl(self) -> str | None:
|
| 686 |
+
return self.url
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
class HTTPResponse(BaseHTTPResponse):
|
| 690 |
+
"""
|
| 691 |
+
HTTP Response container.
|
| 692 |
+
|
| 693 |
+
Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
|
| 694 |
+
loaded and decoded on-demand when the ``data`` property is accessed. This
|
| 695 |
+
class is also compatible with the Python standard library's :mod:`io`
|
| 696 |
+
module, and can hence be treated as a readable object in the context of that
|
| 697 |
+
framework.
|
| 698 |
+
|
| 699 |
+
Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
|
| 700 |
+
|
| 701 |
+
:param preload_content:
|
| 702 |
+
If True, the response's body will be preloaded during construction.
|
| 703 |
+
|
| 704 |
+
:param decode_content:
|
| 705 |
+
If True, will attempt to decode the body based on the
|
| 706 |
+
'content-encoding' header.
|
| 707 |
+
|
| 708 |
+
:param original_response:
|
| 709 |
+
When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
|
| 710 |
+
object, it's convenient to include the original for debug purposes. It's
|
| 711 |
+
otherwise unused.
|
| 712 |
+
|
| 713 |
+
:param retries:
|
| 714 |
+
The retries contains the last :class:`~urllib3.util.retry.Retry` that
|
| 715 |
+
was used during the request.
|
| 716 |
+
|
| 717 |
+
:param enforce_content_length:
|
| 718 |
+
Enforce content length checking. Body returned by server must match
|
| 719 |
+
value of Content-Length header, if present. Otherwise, raise error.
|
| 720 |
+
"""
|
| 721 |
+
|
| 722 |
+
def __init__(
|
| 723 |
+
self,
|
| 724 |
+
body: _TYPE_BODY = "",
|
| 725 |
+
headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
|
| 726 |
+
status: int = 0,
|
| 727 |
+
version: int = 0,
|
| 728 |
+
version_string: str = "HTTP/?",
|
| 729 |
+
reason: str | None = None,
|
| 730 |
+
preload_content: bool = True,
|
| 731 |
+
decode_content: bool = True,
|
| 732 |
+
original_response: _HttplibHTTPResponse | None = None,
|
| 733 |
+
pool: HTTPConnectionPool | None = None,
|
| 734 |
+
connection: HTTPConnection | None = None,
|
| 735 |
+
msg: _HttplibHTTPMessage | None = None,
|
| 736 |
+
retries: Retry | None = None,
|
| 737 |
+
enforce_content_length: bool = True,
|
| 738 |
+
request_method: str | None = None,
|
| 739 |
+
request_url: str | None = None,
|
| 740 |
+
auto_close: bool = True,
|
| 741 |
+
sock_shutdown: typing.Callable[[int], None] | None = None,
|
| 742 |
+
) -> None:
|
| 743 |
+
super().__init__(
|
| 744 |
+
headers=headers,
|
| 745 |
+
status=status,
|
| 746 |
+
version=version,
|
| 747 |
+
version_string=version_string,
|
| 748 |
+
reason=reason,
|
| 749 |
+
decode_content=decode_content,
|
| 750 |
+
request_url=request_url,
|
| 751 |
+
retries=retries,
|
| 752 |
+
)
|
| 753 |
+
|
| 754 |
+
self.enforce_content_length = enforce_content_length
|
| 755 |
+
self.auto_close = auto_close
|
| 756 |
+
|
| 757 |
+
self._body = None
|
| 758 |
+
self._fp: _HttplibHTTPResponse | None = None
|
| 759 |
+
self._original_response = original_response
|
| 760 |
+
self._fp_bytes_read = 0
|
| 761 |
+
self.msg = msg
|
| 762 |
+
|
| 763 |
+
if body and isinstance(body, (str, bytes)):
|
| 764 |
+
self._body = body
|
| 765 |
+
|
| 766 |
+
self._pool = pool
|
| 767 |
+
self._connection = connection
|
| 768 |
+
|
| 769 |
+
if hasattr(body, "read"):
|
| 770 |
+
self._fp = body # type: ignore[assignment]
|
| 771 |
+
self._sock_shutdown = sock_shutdown
|
| 772 |
+
|
| 773 |
+
# Are we using the chunked-style of transfer encoding?
|
| 774 |
+
self.chunk_left: int | None = None
|
| 775 |
+
|
| 776 |
+
# Determine length of response
|
| 777 |
+
self.length_remaining = self._init_length(request_method)
|
| 778 |
+
|
| 779 |
+
# Used to return the correct amount of bytes for partial read()s
|
| 780 |
+
self._decoded_buffer = BytesQueueBuffer()
|
| 781 |
+
|
| 782 |
+
# If requested, preload the body.
|
| 783 |
+
if preload_content and not self._body:
|
| 784 |
+
self._body = self.read(decode_content=decode_content)
|
| 785 |
+
|
| 786 |
+
def release_conn(self) -> None:
|
| 787 |
+
if not self._pool or not self._connection:
|
| 788 |
+
return None
|
| 789 |
+
|
| 790 |
+
self._pool._put_conn(self._connection)
|
| 791 |
+
self._connection = None
|
| 792 |
+
|
| 793 |
+
def drain_conn(self) -> None:
|
| 794 |
+
"""
|
| 795 |
+
Read and discard any remaining HTTP response data in the response connection.
|
| 796 |
+
|
| 797 |
+
Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
|
| 798 |
+
"""
|
| 799 |
+
try:
|
| 800 |
+
self.read(
|
| 801 |
+
# Do not spend resources decoding the content unless
|
| 802 |
+
# decoding has already been initiated.
|
| 803 |
+
decode_content=self._has_decoded_content,
|
| 804 |
+
)
|
| 805 |
+
except (HTTPError, OSError, BaseSSLError, HTTPException):
|
| 806 |
+
pass
|
| 807 |
+
|
| 808 |
+
@property
|
| 809 |
+
def data(self) -> bytes:
|
| 810 |
+
# For backwards-compat with earlier urllib3 0.4 and earlier.
|
| 811 |
+
if self._body:
|
| 812 |
+
return self._body # type: ignore[return-value]
|
| 813 |
+
|
| 814 |
+
if self._fp:
|
| 815 |
+
return self.read(cache_content=True)
|
| 816 |
+
|
| 817 |
+
return None # type: ignore[return-value]
|
| 818 |
+
|
| 819 |
+
@property
|
| 820 |
+
def connection(self) -> HTTPConnection | None:
|
| 821 |
+
return self._connection
|
| 822 |
+
|
| 823 |
+
def isclosed(self) -> bool:
|
| 824 |
+
return is_fp_closed(self._fp)
|
| 825 |
+
|
| 826 |
+
def tell(self) -> int:
|
| 827 |
+
"""
|
| 828 |
+
Obtain the number of bytes pulled over the wire so far. May differ from
|
| 829 |
+
the amount of content returned by :meth:``urllib3.response.HTTPResponse.read``
|
| 830 |
+
if bytes are encoded on the wire (e.g, compressed).
|
| 831 |
+
"""
|
| 832 |
+
return self._fp_bytes_read
|
| 833 |
+
|
| 834 |
+
def _init_length(self, request_method: str | None) -> int | None:
|
| 835 |
+
"""
|
| 836 |
+
Set initial length value for Response content if available.
|
| 837 |
+
"""
|
| 838 |
+
length: int | None
|
| 839 |
+
content_length: str | None = self.headers.get("content-length")
|
| 840 |
+
|
| 841 |
+
if content_length is not None:
|
| 842 |
+
if self.chunked:
|
| 843 |
+
# This Response will fail with an IncompleteRead if it can't be
|
| 844 |
+
# received as chunked. This method falls back to attempt reading
|
| 845 |
+
# the response before raising an exception.
|
| 846 |
+
log.warning(
|
| 847 |
+
"Received response with both Content-Length and "
|
| 848 |
+
"Transfer-Encoding set. This is expressly forbidden "
|
| 849 |
+
"by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
|
| 850 |
+
"attempting to process response as Transfer-Encoding: "
|
| 851 |
+
"chunked."
|
| 852 |
+
)
|
| 853 |
+
return None
|
| 854 |
+
|
| 855 |
+
try:
|
| 856 |
+
# RFC 7230 section 3.3.2 specifies multiple content lengths can
|
| 857 |
+
# be sent in a single Content-Length header
|
| 858 |
+
# (e.g. Content-Length: 42, 42). This line ensures the values
|
| 859 |
+
# are all valid ints and that as long as the `set` length is 1,
|
| 860 |
+
# all values are the same. Otherwise, the header is invalid.
|
| 861 |
+
lengths = {int(val) for val in content_length.split(",")}
|
| 862 |
+
if len(lengths) > 1:
|
| 863 |
+
raise InvalidHeader(
|
| 864 |
+
"Content-Length contained multiple "
|
| 865 |
+
"unmatching values (%s)" % content_length
|
| 866 |
+
)
|
| 867 |
+
length = lengths.pop()
|
| 868 |
+
except ValueError:
|
| 869 |
+
length = None
|
| 870 |
+
else:
|
| 871 |
+
if length < 0:
|
| 872 |
+
length = None
|
| 873 |
+
|
| 874 |
+
else: # if content_length is None
|
| 875 |
+
length = None
|
| 876 |
+
|
| 877 |
+
# Convert status to int for comparison
|
| 878 |
+
# In some cases, httplib returns a status of "_UNKNOWN"
|
| 879 |
+
try:
|
| 880 |
+
status = int(self.status)
|
| 881 |
+
except ValueError:
|
| 882 |
+
status = 0
|
| 883 |
+
|
| 884 |
+
# Check for responses that shouldn't include a body
|
| 885 |
+
if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
|
| 886 |
+
length = 0
|
| 887 |
+
|
| 888 |
+
return length
|
| 889 |
+
|
| 890 |
+
@contextmanager
|
| 891 |
+
def _error_catcher(self) -> typing.Generator[None]:
|
| 892 |
+
"""
|
| 893 |
+
Catch low-level python exceptions, instead re-raising urllib3
|
| 894 |
+
variants, so that low-level exceptions are not leaked in the
|
| 895 |
+
high-level api.
|
| 896 |
+
|
| 897 |
+
On exit, release the connection back to the pool.
|
| 898 |
+
"""
|
| 899 |
+
clean_exit = False
|
| 900 |
+
|
| 901 |
+
try:
|
| 902 |
+
try:
|
| 903 |
+
yield
|
| 904 |
+
|
| 905 |
+
except SocketTimeout as e:
|
| 906 |
+
# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
|
| 907 |
+
# there is yet no clean way to get at it from this context.
|
| 908 |
+
raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
|
| 909 |
+
|
| 910 |
+
except BaseSSLError as e:
|
| 911 |
+
# FIXME: Is there a better way to differentiate between SSLErrors?
|
| 912 |
+
if "read operation timed out" not in str(e):
|
| 913 |
+
# SSL errors related to framing/MAC get wrapped and reraised here
|
| 914 |
+
raise SSLError(e) from e
|
| 915 |
+
|
| 916 |
+
raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type]
|
| 917 |
+
|
| 918 |
+
except IncompleteRead as e:
|
| 919 |
+
if (
|
| 920 |
+
e.expected is not None
|
| 921 |
+
and e.partial is not None
|
| 922 |
+
and e.expected == -e.partial
|
| 923 |
+
):
|
| 924 |
+
arg = "Response may not contain content."
|
| 925 |
+
else:
|
| 926 |
+
arg = f"Connection broken: {e!r}"
|
| 927 |
+
raise ProtocolError(arg, e) from e
|
| 928 |
+
|
| 929 |
+
except (HTTPException, OSError) as e:
|
| 930 |
+
raise ProtocolError(f"Connection broken: {e!r}", e) from e
|
| 931 |
+
|
| 932 |
+
# If no exception is thrown, we should avoid cleaning up
|
| 933 |
+
# unnecessarily.
|
| 934 |
+
clean_exit = True
|
| 935 |
+
finally:
|
| 936 |
+
# If we didn't terminate cleanly, we need to throw away our
|
| 937 |
+
# connection.
|
| 938 |
+
if not clean_exit:
|
| 939 |
+
# The response may not be closed but we're not going to use it
|
| 940 |
+
# anymore so close it now to ensure that the connection is
|
| 941 |
+
# released back to the pool.
|
| 942 |
+
if self._original_response:
|
| 943 |
+
self._original_response.close()
|
| 944 |
+
|
| 945 |
+
# Closing the response may not actually be sufficient to close
|
| 946 |
+
# everything, so if we have a hold of the connection close that
|
| 947 |
+
# too.
|
| 948 |
+
if self._connection:
|
| 949 |
+
self._connection.close()
|
| 950 |
+
|
| 951 |
+
# If we hold the original response but it's closed now, we should
|
| 952 |
+
# return the connection back to the pool.
|
| 953 |
+
if self._original_response and self._original_response.isclosed():
|
| 954 |
+
self.release_conn()
|
| 955 |
+
|
| 956 |
+
def _fp_read(
|
| 957 |
+
self,
|
| 958 |
+
amt: int | None = None,
|
| 959 |
+
*,
|
| 960 |
+
read1: bool = False,
|
| 961 |
+
) -> bytes:
|
| 962 |
+
"""
|
| 963 |
+
Read a response with the thought that reading the number of bytes
|
| 964 |
+
larger than can fit in a 32-bit int at a time via SSL in some
|
| 965 |
+
known cases leads to an overflow error that has to be prevented
|
| 966 |
+
if `amt` or `self.length_remaining` indicate that a problem may
|
| 967 |
+
happen.
|
| 968 |
+
|
| 969 |
+
The known cases:
|
| 970 |
+
* CPython < 3.9.7 because of a bug
|
| 971 |
+
https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900.
|
| 972 |
+
* urllib3 injected with pyOpenSSL-backed SSL-support.
|
| 973 |
+
* CPython < 3.10 only when `amt` does not fit 32-bit int.
|
| 974 |
+
"""
|
| 975 |
+
assert self._fp
|
| 976 |
+
c_int_max = 2**31 - 1
|
| 977 |
+
if (
|
| 978 |
+
(amt and amt > c_int_max)
|
| 979 |
+
or (
|
| 980 |
+
amt is None
|
| 981 |
+
and self.length_remaining
|
| 982 |
+
and self.length_remaining > c_int_max
|
| 983 |
+
)
|
| 984 |
+
) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)):
|
| 985 |
+
if read1:
|
| 986 |
+
return self._fp.read1(c_int_max)
|
| 987 |
+
buffer = io.BytesIO()
|
| 988 |
+
# Besides `max_chunk_amt` being a maximum chunk size, it
|
| 989 |
+
# affects memory overhead of reading a response by this
|
| 990 |
+
# method in CPython.
|
| 991 |
+
# `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
|
| 992 |
+
# chunk size that does not lead to an overflow error, but
|
| 993 |
+
# 256 MiB is a compromise.
|
| 994 |
+
max_chunk_amt = 2**28
|
| 995 |
+
while amt is None or amt != 0:
|
| 996 |
+
if amt is not None:
|
| 997 |
+
chunk_amt = min(amt, max_chunk_amt)
|
| 998 |
+
amt -= chunk_amt
|
| 999 |
+
else:
|
| 1000 |
+
chunk_amt = max_chunk_amt
|
| 1001 |
+
data = self._fp.read(chunk_amt)
|
| 1002 |
+
if not data:
|
| 1003 |
+
break
|
| 1004 |
+
buffer.write(data)
|
| 1005 |
+
del data # to reduce peak memory usage by `max_chunk_amt`.
|
| 1006 |
+
return buffer.getvalue()
|
| 1007 |
+
elif read1:
|
| 1008 |
+
return self._fp.read1(amt) if amt is not None else self._fp.read1()
|
| 1009 |
+
else:
|
| 1010 |
+
# StringIO doesn't like amt=None
|
| 1011 |
+
return self._fp.read(amt) if amt is not None else self._fp.read()
|
| 1012 |
+
|
| 1013 |
+
def _raw_read(
|
| 1014 |
+
self,
|
| 1015 |
+
amt: int | None = None,
|
| 1016 |
+
*,
|
| 1017 |
+
read1: bool = False,
|
| 1018 |
+
) -> bytes:
|
| 1019 |
+
"""
|
| 1020 |
+
Reads `amt` of bytes from the socket.
|
| 1021 |
+
"""
|
| 1022 |
+
if self._fp is None:
|
| 1023 |
+
return None # type: ignore[return-value]
|
| 1024 |
+
|
| 1025 |
+
fp_closed = getattr(self._fp, "closed", False)
|
| 1026 |
+
|
| 1027 |
+
with self._error_catcher():
|
| 1028 |
+
data = self._fp_read(amt, read1=read1) if not fp_closed else b""
|
| 1029 |
+
if amt is not None and amt != 0 and not data:
|
| 1030 |
+
# Platform-specific: Buggy versions of Python.
|
| 1031 |
+
# Close the connection when no data is returned
|
| 1032 |
+
#
|
| 1033 |
+
# This is redundant to what httplib/http.client _should_
|
| 1034 |
+
# already do. However, versions of python released before
|
| 1035 |
+
# December 15, 2012 (http://bugs.python.org/issue16298) do
|
| 1036 |
+
# not properly close the connection in all cases. There is
|
| 1037 |
+
# no harm in redundantly calling close.
|
| 1038 |
+
self._fp.close()
|
| 1039 |
+
if (
|
| 1040 |
+
self.enforce_content_length
|
| 1041 |
+
and self.length_remaining is not None
|
| 1042 |
+
and self.length_remaining != 0
|
| 1043 |
+
):
|
| 1044 |
+
# This is an edge case that httplib failed to cover due
|
| 1045 |
+
# to concerns of backward compatibility. We're
|
| 1046 |
+
# addressing it here to make sure IncompleteRead is
|
| 1047 |
+
# raised during streaming, so all calls with incorrect
|
| 1048 |
+
# Content-Length are caught.
|
| 1049 |
+
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
|
| 1050 |
+
elif read1 and (
|
| 1051 |
+
(amt != 0 and not data) or self.length_remaining == len(data)
|
| 1052 |
+
):
|
| 1053 |
+
# All data has been read, but `self._fp.read1` in
|
| 1054 |
+
# CPython 3.12 and older doesn't always close
|
| 1055 |
+
# `http.client.HTTPResponse`, so we close it here.
|
| 1056 |
+
# See https://github.com/python/cpython/issues/113199
|
| 1057 |
+
self._fp.close()
|
| 1058 |
+
|
| 1059 |
+
if data:
|
| 1060 |
+
self._fp_bytes_read += len(data)
|
| 1061 |
+
if self.length_remaining is not None:
|
| 1062 |
+
self.length_remaining -= len(data)
|
| 1063 |
+
return data
|
| 1064 |
+
|
| 1065 |
+
def read(
|
| 1066 |
+
self,
|
| 1067 |
+
amt: int | None = None,
|
| 1068 |
+
decode_content: bool | None = None,
|
| 1069 |
+
cache_content: bool = False,
|
| 1070 |
+
) -> bytes:
|
| 1071 |
+
"""
|
| 1072 |
+
Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
|
| 1073 |
+
parameters: ``decode_content`` and ``cache_content``.
|
| 1074 |
+
|
| 1075 |
+
:param amt:
|
| 1076 |
+
How much of the content to read. If specified, caching is skipped
|
| 1077 |
+
because it doesn't make sense to cache partial content as the full
|
| 1078 |
+
response.
|
| 1079 |
+
|
| 1080 |
+
:param decode_content:
|
| 1081 |
+
If True, will attempt to decode the body based on the
|
| 1082 |
+
'content-encoding' header.
|
| 1083 |
+
|
| 1084 |
+
:param cache_content:
|
| 1085 |
+
If True, will save the returned data such that the same result is
|
| 1086 |
+
returned despite of the state of the underlying file object. This
|
| 1087 |
+
is useful if you want the ``.data`` property to continue working
|
| 1088 |
+
after having ``.read()`` the file object. (Overridden if ``amt`` is
|
| 1089 |
+
set.)
|
| 1090 |
+
"""
|
| 1091 |
+
self._init_decoder()
|
| 1092 |
+
if decode_content is None:
|
| 1093 |
+
decode_content = self.decode_content
|
| 1094 |
+
|
| 1095 |
+
if amt and amt < 0:
|
| 1096 |
+
# Negative numbers and `None` should be treated the same.
|
| 1097 |
+
amt = None
|
| 1098 |
+
elif amt is not None:
|
| 1099 |
+
cache_content = False
|
| 1100 |
+
|
| 1101 |
+
if self._decoder and self._decoder.has_unconsumed_tail:
|
| 1102 |
+
decoded_data = self._decode(
|
| 1103 |
+
b"",
|
| 1104 |
+
decode_content,
|
| 1105 |
+
flush_decoder=False,
|
| 1106 |
+
max_length=amt - len(self._decoded_buffer),
|
| 1107 |
+
)
|
| 1108 |
+
self._decoded_buffer.put(decoded_data)
|
| 1109 |
+
if len(self._decoded_buffer) >= amt:
|
| 1110 |
+
return self._decoded_buffer.get(amt)
|
| 1111 |
+
|
| 1112 |
+
data = self._raw_read(amt)
|
| 1113 |
+
|
| 1114 |
+
flush_decoder = amt is None or (amt != 0 and not data)
|
| 1115 |
+
|
| 1116 |
+
if (
|
| 1117 |
+
not data
|
| 1118 |
+
and len(self._decoded_buffer) == 0
|
| 1119 |
+
and not (self._decoder and self._decoder.has_unconsumed_tail)
|
| 1120 |
+
):
|
| 1121 |
+
return data
|
| 1122 |
+
|
| 1123 |
+
if amt is None:
|
| 1124 |
+
data = self._decode(data, decode_content, flush_decoder)
|
| 1125 |
+
if cache_content:
|
| 1126 |
+
self._body = data
|
| 1127 |
+
else:
|
| 1128 |
+
# do not waste memory on buffer when not decoding
|
| 1129 |
+
if not decode_content:
|
| 1130 |
+
if self._has_decoded_content:
|
| 1131 |
+
raise RuntimeError(
|
| 1132 |
+
"Calling read(decode_content=False) is not supported after "
|
| 1133 |
+
"read(decode_content=True) was called."
|
| 1134 |
+
)
|
| 1135 |
+
return data
|
| 1136 |
+
|
| 1137 |
+
decoded_data = self._decode(
|
| 1138 |
+
data,
|
| 1139 |
+
decode_content,
|
| 1140 |
+
flush_decoder,
|
| 1141 |
+
max_length=amt - len(self._decoded_buffer),
|
| 1142 |
+
)
|
| 1143 |
+
self._decoded_buffer.put(decoded_data)
|
| 1144 |
+
|
| 1145 |
+
while len(self._decoded_buffer) < amt and data:
|
| 1146 |
+
# TODO make sure to initially read enough data to get past the headers
|
| 1147 |
+
# For example, the GZ file header takes 10 bytes, we don't want to read
|
| 1148 |
+
# it one byte at a time
|
| 1149 |
+
data = self._raw_read(amt)
|
| 1150 |
+
decoded_data = self._decode(
|
| 1151 |
+
data,
|
| 1152 |
+
decode_content,
|
| 1153 |
+
flush_decoder,
|
| 1154 |
+
max_length=amt - len(self._decoded_buffer),
|
| 1155 |
+
)
|
| 1156 |
+
self._decoded_buffer.put(decoded_data)
|
| 1157 |
+
data = self._decoded_buffer.get(amt)
|
| 1158 |
+
|
| 1159 |
+
return data
|
| 1160 |
+
|
| 1161 |
+
def read1(
|
| 1162 |
+
self,
|
| 1163 |
+
amt: int | None = None,
|
| 1164 |
+
decode_content: bool | None = None,
|
| 1165 |
+
) -> bytes:
|
| 1166 |
+
"""
|
| 1167 |
+
Similar to ``http.client.HTTPResponse.read1`` and documented
|
| 1168 |
+
in :meth:`io.BufferedReader.read1`, but with an additional parameter:
|
| 1169 |
+
``decode_content``.
|
| 1170 |
+
|
| 1171 |
+
:param amt:
|
| 1172 |
+
How much of the content to read.
|
| 1173 |
+
|
| 1174 |
+
:param decode_content:
|
| 1175 |
+
If True, will attempt to decode the body based on the
|
| 1176 |
+
'content-encoding' header.
|
| 1177 |
+
"""
|
| 1178 |
+
if decode_content is None:
|
| 1179 |
+
decode_content = self.decode_content
|
| 1180 |
+
if amt and amt < 0:
|
| 1181 |
+
# Negative numbers and `None` should be treated the same.
|
| 1182 |
+
amt = None
|
| 1183 |
+
# try and respond without going to the network
|
| 1184 |
+
if self._has_decoded_content:
|
| 1185 |
+
if not decode_content:
|
| 1186 |
+
raise RuntimeError(
|
| 1187 |
+
"Calling read1(decode_content=False) is not supported after "
|
| 1188 |
+
"read1(decode_content=True) was called."
|
| 1189 |
+
)
|
| 1190 |
+
if (
|
| 1191 |
+
self._decoder
|
| 1192 |
+
and self._decoder.has_unconsumed_tail
|
| 1193 |
+
and (amt is None or len(self._decoded_buffer) < amt)
|
| 1194 |
+
):
|
| 1195 |
+
decoded_data = self._decode(
|
| 1196 |
+
b"",
|
| 1197 |
+
decode_content,
|
| 1198 |
+
flush_decoder=False,
|
| 1199 |
+
max_length=(
|
| 1200 |
+
amt - len(self._decoded_buffer) if amt is not None else None
|
| 1201 |
+
),
|
| 1202 |
+
)
|
| 1203 |
+
self._decoded_buffer.put(decoded_data)
|
| 1204 |
+
if len(self._decoded_buffer) > 0:
|
| 1205 |
+
if amt is None:
|
| 1206 |
+
return self._decoded_buffer.get_all()
|
| 1207 |
+
return self._decoded_buffer.get(amt)
|
| 1208 |
+
if amt == 0:
|
| 1209 |
+
return b""
|
| 1210 |
+
|
| 1211 |
+
# FIXME, this method's type doesn't say returning None is possible
|
| 1212 |
+
data = self._raw_read(amt, read1=True)
|
| 1213 |
+
if not decode_content or data is None:
|
| 1214 |
+
return data
|
| 1215 |
+
|
| 1216 |
+
self._init_decoder()
|
| 1217 |
+
while True:
|
| 1218 |
+
flush_decoder = not data
|
| 1219 |
+
decoded_data = self._decode(
|
| 1220 |
+
data, decode_content, flush_decoder, max_length=amt
|
| 1221 |
+
)
|
| 1222 |
+
self._decoded_buffer.put(decoded_data)
|
| 1223 |
+
if decoded_data or flush_decoder:
|
| 1224 |
+
break
|
| 1225 |
+
data = self._raw_read(8192, read1=True)
|
| 1226 |
+
|
| 1227 |
+
if amt is None:
|
| 1228 |
+
return self._decoded_buffer.get_all()
|
| 1229 |
+
return self._decoded_buffer.get(amt)
|
| 1230 |
+
|
| 1231 |
+
def stream(
|
| 1232 |
+
self, amt: int | None = 2**16, decode_content: bool | None = None
|
| 1233 |
+
) -> typing.Generator[bytes]:
|
| 1234 |
+
"""
|
| 1235 |
+
A generator wrapper for the read() method. A call will block until
|
| 1236 |
+
``amt`` bytes have been read from the connection or until the
|
| 1237 |
+
connection is closed.
|
| 1238 |
+
|
| 1239 |
+
:param amt:
|
| 1240 |
+
How much of the content to read. The generator will return up to
|
| 1241 |
+
much data per iteration, but may return less. This is particularly
|
| 1242 |
+
likely when using compressed data. However, the empty string will
|
| 1243 |
+
never be returned.
|
| 1244 |
+
|
| 1245 |
+
:param decode_content:
|
| 1246 |
+
If True, will attempt to decode the body based on the
|
| 1247 |
+
'content-encoding' header.
|
| 1248 |
+
"""
|
| 1249 |
+
if self.chunked and self.supports_chunked_reads():
|
| 1250 |
+
yield from self.read_chunked(amt, decode_content=decode_content)
|
| 1251 |
+
else:
|
| 1252 |
+
while (
|
| 1253 |
+
not is_fp_closed(self._fp)
|
| 1254 |
+
or len(self._decoded_buffer) > 0
|
| 1255 |
+
or (self._decoder and self._decoder.has_unconsumed_tail)
|
| 1256 |
+
):
|
| 1257 |
+
data = self.read(amt=amt, decode_content=decode_content)
|
| 1258 |
+
|
| 1259 |
+
if data:
|
| 1260 |
+
yield data
|
| 1261 |
+
|
| 1262 |
+
# Overrides from io.IOBase
|
| 1263 |
+
def readable(self) -> bool:
|
| 1264 |
+
return True
|
| 1265 |
+
|
| 1266 |
+
def shutdown(self) -> None:
|
| 1267 |
+
if not self._sock_shutdown:
|
| 1268 |
+
raise ValueError("Cannot shutdown socket as self._sock_shutdown is not set")
|
| 1269 |
+
if self._connection is None:
|
| 1270 |
+
raise RuntimeError(
|
| 1271 |
+
"Cannot shutdown as connection has already been released to the pool"
|
| 1272 |
+
)
|
| 1273 |
+
self._sock_shutdown(socket.SHUT_RD)
|
| 1274 |
+
|
| 1275 |
+
def close(self) -> None:
|
| 1276 |
+
self._sock_shutdown = None
|
| 1277 |
+
|
| 1278 |
+
if not self.closed and self._fp:
|
| 1279 |
+
self._fp.close()
|
| 1280 |
+
|
| 1281 |
+
if self._connection:
|
| 1282 |
+
self._connection.close()
|
| 1283 |
+
|
| 1284 |
+
if not self.auto_close:
|
| 1285 |
+
io.IOBase.close(self)
|
| 1286 |
+
|
| 1287 |
+
@property
|
| 1288 |
+
def closed(self) -> bool:
|
| 1289 |
+
if not self.auto_close:
|
| 1290 |
+
return io.IOBase.closed.__get__(self) # type: ignore[no-any-return]
|
| 1291 |
+
elif self._fp is None:
|
| 1292 |
+
return True
|
| 1293 |
+
elif hasattr(self._fp, "isclosed"):
|
| 1294 |
+
return self._fp.isclosed()
|
| 1295 |
+
elif hasattr(self._fp, "closed"):
|
| 1296 |
+
return self._fp.closed
|
| 1297 |
+
else:
|
| 1298 |
+
return True
|
| 1299 |
+
|
| 1300 |
+
def fileno(self) -> int:
|
| 1301 |
+
if self._fp is None:
|
| 1302 |
+
raise OSError("HTTPResponse has no file to get a fileno from")
|
| 1303 |
+
elif hasattr(self._fp, "fileno"):
|
| 1304 |
+
return self._fp.fileno()
|
| 1305 |
+
else:
|
| 1306 |
+
raise OSError(
|
| 1307 |
+
"The file-like object this HTTPResponse is wrapped "
|
| 1308 |
+
"around has no file descriptor"
|
| 1309 |
+
)
|
| 1310 |
+
|
| 1311 |
+
def flush(self) -> None:
|
| 1312 |
+
if (
|
| 1313 |
+
self._fp is not None
|
| 1314 |
+
and hasattr(self._fp, "flush")
|
| 1315 |
+
and not getattr(self._fp, "closed", False)
|
| 1316 |
+
):
|
| 1317 |
+
return self._fp.flush()
|
| 1318 |
+
|
| 1319 |
+
def supports_chunked_reads(self) -> bool:
|
| 1320 |
+
"""
|
| 1321 |
+
Checks if the underlying file-like object looks like a
|
| 1322 |
+
:class:`http.client.HTTPResponse` object. We do this by testing for
|
| 1323 |
+
the fp attribute. If it is present we assume it returns raw chunks as
|
| 1324 |
+
processed by read_chunked().
|
| 1325 |
+
"""
|
| 1326 |
+
return hasattr(self._fp, "fp")
|
| 1327 |
+
|
| 1328 |
+
def _update_chunk_length(self) -> None:
|
| 1329 |
+
# First, we'll figure out length of a chunk and then
|
| 1330 |
+
# we'll try to read it from socket.
|
| 1331 |
+
if self.chunk_left is not None:
|
| 1332 |
+
return None
|
| 1333 |
+
line = self._fp.fp.readline() # type: ignore[union-attr]
|
| 1334 |
+
line = line.split(b";", 1)[0]
|
| 1335 |
+
try:
|
| 1336 |
+
self.chunk_left = int(line, 16)
|
| 1337 |
+
except ValueError:
|
| 1338 |
+
self.close()
|
| 1339 |
+
if line:
|
| 1340 |
+
# Invalid chunked protocol response, abort.
|
| 1341 |
+
raise InvalidChunkLength(self, line) from None
|
| 1342 |
+
else:
|
| 1343 |
+
# Truncated at start of next chunk
|
| 1344 |
+
raise ProtocolError("Response ended prematurely") from None
|
| 1345 |
+
|
| 1346 |
+
def _handle_chunk(self, amt: int | None) -> bytes:
|
| 1347 |
+
returned_chunk = None
|
| 1348 |
+
if amt is None:
|
| 1349 |
+
chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
|
| 1350 |
+
returned_chunk = chunk
|
| 1351 |
+
self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
|
| 1352 |
+
self.chunk_left = None
|
| 1353 |
+
elif self.chunk_left is not None and amt < self.chunk_left:
|
| 1354 |
+
value = self._fp._safe_read(amt) # type: ignore[union-attr]
|
| 1355 |
+
self.chunk_left = self.chunk_left - amt
|
| 1356 |
+
returned_chunk = value
|
| 1357 |
+
elif amt == self.chunk_left:
|
| 1358 |
+
value = self._fp._safe_read(amt) # type: ignore[union-attr]
|
| 1359 |
+
self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
|
| 1360 |
+
self.chunk_left = None
|
| 1361 |
+
returned_chunk = value
|
| 1362 |
+
else: # amt > self.chunk_left
|
| 1363 |
+
returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr]
|
| 1364 |
+
self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
|
| 1365 |
+
self.chunk_left = None
|
| 1366 |
+
return returned_chunk # type: ignore[no-any-return]
|
| 1367 |
+
|
| 1368 |
+
def read_chunked(
|
| 1369 |
+
self, amt: int | None = None, decode_content: bool | None = None
|
| 1370 |
+
) -> typing.Generator[bytes]:
|
| 1371 |
+
"""
|
| 1372 |
+
Similar to :meth:`HTTPResponse.read`, but with an additional
|
| 1373 |
+
parameter: ``decode_content``.
|
| 1374 |
+
|
| 1375 |
+
:param amt:
|
| 1376 |
+
How much of the content to read. If specified, caching is skipped
|
| 1377 |
+
because it doesn't make sense to cache partial content as the full
|
| 1378 |
+
response.
|
| 1379 |
+
|
| 1380 |
+
:param decode_content:
|
| 1381 |
+
If True, will attempt to decode the body based on the
|
| 1382 |
+
'content-encoding' header.
|
| 1383 |
+
"""
|
| 1384 |
+
self._init_decoder()
|
| 1385 |
+
# FIXME: Rewrite this method and make it a class with a better structured logic.
|
| 1386 |
+
if not self.chunked:
|
| 1387 |
+
raise ResponseNotChunked(
|
| 1388 |
+
"Response is not chunked. "
|
| 1389 |
+
"Header 'transfer-encoding: chunked' is missing."
|
| 1390 |
+
)
|
| 1391 |
+
if not self.supports_chunked_reads():
|
| 1392 |
+
raise BodyNotHttplibCompatible(
|
| 1393 |
+
"Body should be http.client.HTTPResponse like. "
|
| 1394 |
+
"It should have have an fp attribute which returns raw chunks."
|
| 1395 |
+
)
|
| 1396 |
+
|
| 1397 |
+
with self._error_catcher():
|
| 1398 |
+
# Don't bother reading the body of a HEAD request.
|
| 1399 |
+
if self._original_response and is_response_to_head(self._original_response):
|
| 1400 |
+
self._original_response.close()
|
| 1401 |
+
return None
|
| 1402 |
+
|
| 1403 |
+
# If a response is already read and closed
|
| 1404 |
+
# then return immediately.
|
| 1405 |
+
if self._fp.fp is None: # type: ignore[union-attr]
|
| 1406 |
+
return None
|
| 1407 |
+
|
| 1408 |
+
if amt and amt < 0:
|
| 1409 |
+
# Negative numbers and `None` should be treated the same,
|
| 1410 |
+
# but httplib handles only `None` correctly.
|
| 1411 |
+
amt = None
|
| 1412 |
+
|
| 1413 |
+
while True:
|
| 1414 |
+
# First, check if any data is left in the decoder's buffer.
|
| 1415 |
+
if self._decoder and self._decoder.has_unconsumed_tail:
|
| 1416 |
+
chunk = b""
|
| 1417 |
+
else:
|
| 1418 |
+
self._update_chunk_length()
|
| 1419 |
+
if self.chunk_left == 0:
|
| 1420 |
+
break
|
| 1421 |
+
chunk = self._handle_chunk(amt)
|
| 1422 |
+
decoded = self._decode(
|
| 1423 |
+
chunk,
|
| 1424 |
+
decode_content=decode_content,
|
| 1425 |
+
flush_decoder=False,
|
| 1426 |
+
max_length=amt,
|
| 1427 |
+
)
|
| 1428 |
+
if decoded:
|
| 1429 |
+
yield decoded
|
| 1430 |
+
|
| 1431 |
+
if decode_content:
|
| 1432 |
+
# On CPython and PyPy, we should never need to flush the
|
| 1433 |
+
# decoder. However, on Jython we *might* need to, so
|
| 1434 |
+
# lets defensively do it anyway.
|
| 1435 |
+
decoded = self._flush_decoder()
|
| 1436 |
+
if decoded: # Platform-specific: Jython.
|
| 1437 |
+
yield decoded
|
| 1438 |
+
|
| 1439 |
+
# Chunk content ends with \r\n: discard it.
|
| 1440 |
+
while self._fp is not None:
|
| 1441 |
+
line = self._fp.fp.readline()
|
| 1442 |
+
if not line:
|
| 1443 |
+
# Some sites may not end with '\r\n'.
|
| 1444 |
+
break
|
| 1445 |
+
if line == b"\r\n":
|
| 1446 |
+
break
|
| 1447 |
+
|
| 1448 |
+
# We read everything; close the "file".
|
| 1449 |
+
if self._original_response:
|
| 1450 |
+
self._original_response.close()
|
| 1451 |
+
|
| 1452 |
+
@property
|
| 1453 |
+
def url(self) -> str | None:
|
| 1454 |
+
"""
|
| 1455 |
+
Returns the URL that was the source of this response.
|
| 1456 |
+
If the request that generated this response redirected, this method
|
| 1457 |
+
will return the final redirect location.
|
| 1458 |
+
"""
|
| 1459 |
+
return self._request_url
|
| 1460 |
+
|
| 1461 |
+
@url.setter
|
| 1462 |
+
def url(self, url: str | None) -> None:
|
| 1463 |
+
self._request_url = url
|
| 1464 |
+
|
| 1465 |
+
def __iter__(self) -> typing.Iterator[bytes]:
|
| 1466 |
+
buffer: list[bytes] = []
|
| 1467 |
+
for chunk in self.stream(decode_content=True):
|
| 1468 |
+
if b"\n" in chunk:
|
| 1469 |
+
chunks = chunk.split(b"\n")
|
| 1470 |
+
yield b"".join(buffer) + chunks[0] + b"\n"
|
| 1471 |
+
for x in chunks[1:-1]:
|
| 1472 |
+
yield x + b"\n"
|
| 1473 |
+
if chunks[-1]:
|
| 1474 |
+
buffer = [chunks[-1]]
|
| 1475 |
+
else:
|
| 1476 |
+
buffer = []
|
| 1477 |
+
else:
|
| 1478 |
+
buffer.append(chunk)
|
| 1479 |
+
if buffer:
|
| 1480 |
+
yield b"".join(buffer)
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__init__.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# For backwards compatibility, provide imports that used to be here.
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from .connection import is_connection_dropped
|
| 5 |
+
from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
|
| 6 |
+
from .response import is_fp_closed
|
| 7 |
+
from .retry import Retry
|
| 8 |
+
from .ssl_ import (
|
| 9 |
+
ALPN_PROTOCOLS,
|
| 10 |
+
IS_PYOPENSSL,
|
| 11 |
+
SSLContext,
|
| 12 |
+
assert_fingerprint,
|
| 13 |
+
create_urllib3_context,
|
| 14 |
+
resolve_cert_reqs,
|
| 15 |
+
resolve_ssl_version,
|
| 16 |
+
ssl_wrap_socket,
|
| 17 |
+
)
|
| 18 |
+
from .timeout import Timeout
|
| 19 |
+
from .url import Url, parse_url
|
| 20 |
+
from .wait import wait_for_read, wait_for_write
|
| 21 |
+
|
| 22 |
+
__all__ = (
|
| 23 |
+
"IS_PYOPENSSL",
|
| 24 |
+
"SSLContext",
|
| 25 |
+
"ALPN_PROTOCOLS",
|
| 26 |
+
"Retry",
|
| 27 |
+
"Timeout",
|
| 28 |
+
"Url",
|
| 29 |
+
"assert_fingerprint",
|
| 30 |
+
"create_urllib3_context",
|
| 31 |
+
"is_connection_dropped",
|
| 32 |
+
"is_fp_closed",
|
| 33 |
+
"parse_url",
|
| 34 |
+
"make_headers",
|
| 35 |
+
"resolve_cert_reqs",
|
| 36 |
+
"resolve_ssl_version",
|
| 37 |
+
"ssl_wrap_socket",
|
| 38 |
+
"wait_for_read",
|
| 39 |
+
"wait_for_write",
|
| 40 |
+
"SKIP_HEADER",
|
| 41 |
+
"SKIPPABLE_HEADERS",
|
| 42 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/__init__.cpython-312.pyc
ADDED
|
Binary file (1.01 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/connection.cpython-312.pyc
ADDED
|
Binary file (4.69 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/proxy.cpython-312.pyc
ADDED
|
Binary file (1.22 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/request.cpython-312.pyc
ADDED
|
Binary file (8.34 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/response.cpython-312.pyc
ADDED
|
Binary file (2.88 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/retry.cpython-312.pyc
ADDED
|
Binary file (20.8 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/ssl_.cpython-312.pyc
ADDED
|
Binary file (17.3 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/urllib3/util/__pycache__/ssl_match_hostname.cpython-312.pyc
ADDED
|
Binary file (5.54 kB). View file
|
|
|