diff --git a/.venv/lib/python3.11/site-packages/_yaml/__init__.py b/.venv/lib/python3.11/site-packages/_yaml/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7baa8c4b68127d5cdf0be9a799429e61347c2694 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/_yaml/__init__.py @@ -0,0 +1,33 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +# in some circumstances, the yaml module we imoprted may be from a different version, so we need +# to tread carefully when poking at it here (it may not have the attributes we expect) +if not getattr(yaml, '__with_libyaml__', False): + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/.venv/lib/python3.11/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9581e44af752ea67aeabdbb80e38e98eb77651f7 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/_yaml/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/aiosignal/__init__.py b/.venv/lib/python3.11/site-packages/aiosignal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4ad0278993a986cecabf966d7b6ffd0c58f060e4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/aiosignal/__init__.py @@ -0,0 +1,36 @@ +from frozenlist import FrozenList + +__version__ = "1.3.2" + +__all__ = ("Signal",) + + +class Signal(FrozenList): + """Coroutine-based signal implementation. + + To connect a callback to a signal, use any list method. + + Signals are fired using the send() coroutine, which takes named + arguments. + """ + + __slots__ = ("_owner",) + + def __init__(self, owner): + super().__init__() + self._owner = owner + + def __repr__(self): + return "".format( + self._owner, self.frozen, list(self) + ) + + async def send(self, *args, **kwargs): + """ + Sends data to all registered receivers. + """ + if not self.frozen: + raise RuntimeError("Cannot send non-frozen signal.") + + for receiver in self: + await receiver(*args, **kwargs) # type: ignore diff --git a/.venv/lib/python3.11/site-packages/aiosignal/__init__.pyi b/.venv/lib/python3.11/site-packages/aiosignal/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..d4e3416d72246058259061578a82697e2bc0706e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/aiosignal/__init__.pyi @@ -0,0 +1,12 @@ +from typing import Any, Generic, TypeVar + +from frozenlist import FrozenList + +__all__ = ("Signal",) + +_T = TypeVar("_T") + +class Signal(FrozenList[_T], Generic[_T]): + def __init__(self, owner: Any) -> None: ... + def __repr__(self) -> str: ... + async def send(self, *args: Any, **kwargs: Any) -> None: ... diff --git a/.venv/lib/python3.11/site-packages/aiosignal/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/aiosignal/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..efd2b2b9a49531fe1c64593ac375e5eb6e47c157 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/aiosignal/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/aiosignal/py.typed b/.venv/lib/python3.11/site-packages/aiosignal/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/aead.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/aead.pyi new file mode 100644 index 0000000000000000000000000000000000000000..047f49d819c17565b454ab66aac650c15ae2cd32 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/aead.pyi @@ -0,0 +1,103 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +class AESGCM: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_key(key_size: int) -> bytes: ... + def encrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + def decrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + +class ChaCha20Poly1305: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_key() -> bytes: ... + def encrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + def decrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + +class AESCCM: + def __init__(self, key: bytes, tag_length: int = 16) -> None: ... + @staticmethod + def generate_key(key_size: int) -> bytes: ... + def encrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + def decrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + +class AESSIV: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_key(key_size: int) -> bytes: ... + def encrypt( + self, + data: bytes, + associated_data: list[bytes] | None, + ) -> bytes: ... + def decrypt( + self, + data: bytes, + associated_data: list[bytes] | None, + ) -> bytes: ... + +class AESOCB3: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_key(key_size: int) -> bytes: ... + def encrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + def decrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + +class AESGCMSIV: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_key(key_size: int) -> bytes: ... + def encrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... + def decrypt( + self, + nonce: bytes, + data: bytes, + associated_data: bytes | None, + ) -> bytes: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9c03508bc89bf087e24c684802eeec344d76bb9e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/cmac.pyi @@ -0,0 +1,18 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import ciphers + +class CMAC: + def __init__( + self, + algorithm: ciphers.BlockCipherAlgorithm, + backend: typing.Any = None, + ) -> None: ... + def update(self, data: bytes) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> CMAC: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ec.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ec.pyi new file mode 100644 index 0000000000000000000000000000000000000000..5c3b7bf6e4a9bafac1423bd3936ebc347885a950 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ec.pyi @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import ec + +class ECPrivateKey: ... +class ECPublicKey: ... + +class EllipticCurvePrivateNumbers: + def __init__( + self, private_value: int, public_numbers: EllipticCurvePublicNumbers + ) -> None: ... + def private_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePrivateKey: ... + @property + def private_value(self) -> int: ... + @property + def public_numbers(self) -> EllipticCurvePublicNumbers: ... + +class EllipticCurvePublicNumbers: + def __init__(self, x: int, y: int, curve: ec.EllipticCurve) -> None: ... + def public_key( + self, backend: typing.Any = None + ) -> ec.EllipticCurvePublicKey: ... + @property + def x(self) -> int: ... + @property + def y(self) -> int: ... + @property + def curve(self) -> ec.EllipticCurve: ... + def __eq__(self, other: object) -> bool: ... + +def curve_supported(curve: ec.EllipticCurve) -> bool: ... +def generate_private_key( + curve: ec.EllipticCurve, backend: typing.Any = None +) -> ec.EllipticCurvePrivateKey: ... +def from_private_numbers( + numbers: ec.EllipticCurvePrivateNumbers, +) -> ec.EllipticCurvePrivateKey: ... +def from_public_numbers( + numbers: ec.EllipticCurvePublicNumbers, +) -> ec.EllipticCurvePublicKey: ... +def from_public_bytes( + curve: ec.EllipticCurve, data: bytes +) -> ec.EllipticCurvePublicKey: ... +def derive_private_key( + private_value: int, curve: ec.EllipticCurve +) -> ec.EllipticCurvePrivateKey: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi new file mode 100644 index 0000000000000000000000000000000000000000..5233f9a1d1c8e45f928bd4b41cf1e821332a2a67 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed25519.pyi @@ -0,0 +1,12 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed25519 + +class Ed25519PrivateKey: ... +class Ed25519PublicKey: ... + +def generate_key() -> ed25519.Ed25519PrivateKey: ... +def from_private_bytes(data: bytes) -> ed25519.Ed25519PrivateKey: ... +def from_public_bytes(data: bytes) -> ed25519.Ed25519PublicKey: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi new file mode 100644 index 0000000000000000000000000000000000000000..7a06520380a08886f281a5115ca07bb34004eca1 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/ed448.pyi @@ -0,0 +1,12 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.asymmetric import ed448 + +class Ed448PrivateKey: ... +class Ed448PublicKey: ... + +def generate_key() -> ed448.Ed448PrivateKey: ... +def from_private_bytes(data: bytes) -> ed448.Ed448PrivateKey: ... +def from_public_bytes(data: bytes) -> ed448.Ed448PublicKey: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi new file mode 100644 index 0000000000000000000000000000000000000000..e38d9b54d01bc6e5baea24c2ac6c3d8631941f80 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/hmac.pyi @@ -0,0 +1,21 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives import hashes + +class HMAC(hashes.HashContext): + def __init__( + self, + key: bytes, + algorithm: hashes.HashAlgorithm, + backend: typing.Any = None, + ) -> None: ... + @property + def algorithm(self) -> hashes.HashAlgorithm: ... + def update(self, data: bytes) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, signature: bytes) -> None: ... + def copy(self) -> HMAC: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi new file mode 100644 index 0000000000000000000000000000000000000000..034a8fed2e7898baa5acb56ce01d96ba9fcbb89e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/kdf.pyi @@ -0,0 +1,22 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from cryptography.hazmat.primitives.hashes import HashAlgorithm + +def derive_pbkdf2_hmac( + key_material: bytes, + algorithm: HashAlgorithm, + salt: bytes, + iterations: int, + length: int, +) -> bytes: ... +def derive_scrypt( + key_material: bytes, + salt: bytes, + n: int, + r: int, + p: int, + max_mem: int, + length: int, +) -> bytes: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2e9b0a9e1254361860942722a03281e9bb8a3bb3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/poly1305.pyi @@ -0,0 +1,13 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +class Poly1305: + def __init__(self, key: bytes) -> None: ... + @staticmethod + def generate_tag(key: bytes, data: bytes) -> bytes: ... + @staticmethod + def verify_tag(key: bytes, data: bytes, tag: bytes) -> None: ... + def update(self, data: bytes) -> None: ... + def finalize(self) -> bytes: ... + def verify(self, tag: bytes) -> None: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi new file mode 100644 index 0000000000000000000000000000000000000000..ef7752ddb79db62489bcf71df933fdefd83c0f9d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/_rust/openssl/rsa.pyi @@ -0,0 +1,55 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +import typing + +from cryptography.hazmat.primitives.asymmetric import rsa + +class RSAPrivateKey: ... +class RSAPublicKey: ... + +class RSAPrivateNumbers: + def __init__( + self, + p: int, + q: int, + d: int, + dmp1: int, + dmq1: int, + iqmp: int, + public_numbers: RSAPublicNumbers, + ) -> None: ... + @property + def p(self) -> int: ... + @property + def q(self) -> int: ... + @property + def d(self) -> int: ... + @property + def dmp1(self) -> int: ... + @property + def dmq1(self) -> int: ... + @property + def iqmp(self) -> int: ... + @property + def public_numbers(self) -> RSAPublicNumbers: ... + def private_key( + self, + backend: typing.Any = None, + *, + unsafe_skip_rsa_key_validation: bool = False, + ) -> rsa.RSAPrivateKey: ... + +class RSAPublicNumbers: + def __init__(self, e: int, n: int) -> None: ... + @property + def n(self) -> int: ... + @property + def e(self) -> int: ... + def public_key(self, backend: typing.Any = None) -> rsa.RSAPublicKey: ... + +def generate_private_key( + public_exponent: int, + key_size: int, +) -> rsa.RSAPrivateKey: ... diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__init__.py b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b509336233c2fafe4185a49da5909c8bbb38dfd7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__init__.py @@ -0,0 +1,3 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..435e0a02bb7e6567545ec232be65468df269d485 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-311.pyc b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65897104f368dc541a2faaf745bfb02adb7c7db6 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-311.pyc b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3cd52ce5cf98729f92dc4a4e4ae6117b5c135a1 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py new file mode 100644 index 0000000000000000000000000000000000000000..73c06f7d08ce78610a77e7f702ede6de54011334 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py @@ -0,0 +1,183 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + + +def cryptography_has_set_cert_cb() -> list[str]: + return [ + "SSL_CTX_set_cert_cb", + "SSL_set_cert_cb", + ] + + +def cryptography_has_ssl_st() -> list[str]: + return [ + "SSL_ST_BEFORE", + "SSL_ST_OK", + "SSL_ST_INIT", + "SSL_ST_RENEGOTIATE", + ] + + +def cryptography_has_tls_st() -> list[str]: + return [ + "TLS_ST_BEFORE", + "TLS_ST_OK", + ] + + +def cryptography_has_ssl_sigalgs() -> list[str]: + return [ + "SSL_CTX_set1_sigalgs_list", + ] + + +def cryptography_has_psk() -> list[str]: + return [ + "SSL_CTX_use_psk_identity_hint", + "SSL_CTX_set_psk_server_callback", + "SSL_CTX_set_psk_client_callback", + ] + + +def cryptography_has_psk_tlsv13() -> list[str]: + return [ + "SSL_CTX_set_psk_find_session_callback", + "SSL_CTX_set_psk_use_session_callback", + "Cryptography_SSL_SESSION_new", + "SSL_CIPHER_find", + "SSL_SESSION_set1_master_key", + "SSL_SESSION_set_cipher", + "SSL_SESSION_set_protocol_version", + ] + + +def cryptography_has_custom_ext() -> list[str]: + return [ + "SSL_CTX_add_client_custom_ext", + "SSL_CTX_add_server_custom_ext", + "SSL_extension_supported", + ] + + +def cryptography_has_tlsv13_functions() -> list[str]: + return [ + "SSL_VERIFY_POST_HANDSHAKE", + "SSL_CTX_set_ciphersuites", + "SSL_verify_client_post_handshake", + "SSL_CTX_set_post_handshake_auth", + "SSL_set_post_handshake_auth", + "SSL_SESSION_get_max_early_data", + "SSL_write_early_data", + "SSL_read_early_data", + "SSL_CTX_set_max_early_data", + ] + + +def cryptography_has_engine() -> list[str]: + return [ + "ENGINE_by_id", + "ENGINE_init", + "ENGINE_finish", + "ENGINE_get_default_RAND", + "ENGINE_set_default_RAND", + "ENGINE_unregister_RAND", + "ENGINE_ctrl_cmd", + "ENGINE_free", + "ENGINE_get_name", + "ENGINE_ctrl_cmd_string", + "ENGINE_load_builtin_engines", + "ENGINE_load_private_key", + "ENGINE_load_public_key", + "SSL_CTX_set_client_cert_engine", + ] + + +def cryptography_has_verified_chain() -> list[str]: + return [ + "SSL_get0_verified_chain", + ] + + +def cryptography_has_srtp() -> list[str]: + return [ + "SSL_CTX_set_tlsext_use_srtp", + "SSL_set_tlsext_use_srtp", + "SSL_get_selected_srtp_profile", + ] + + +def cryptography_has_op_no_renegotiation() -> list[str]: + return [ + "SSL_OP_NO_RENEGOTIATION", + ] + + +def cryptography_has_dtls_get_data_mtu() -> list[str]: + return [ + "DTLS_get_data_mtu", + ] + + +def cryptography_has_ssl_cookie() -> list[str]: + return [ + "SSL_OP_COOKIE_EXCHANGE", + "DTLSv1_listen", + "SSL_CTX_set_cookie_generate_cb", + "SSL_CTX_set_cookie_verify_cb", + ] + + +def cryptography_has_prime_checks() -> list[str]: + return [ + "BN_prime_checks_for_size", + ] + + +def cryptography_has_unexpected_eof_while_reading() -> list[str]: + return ["SSL_R_UNEXPECTED_EOF_WHILE_READING"] + + +def cryptography_has_ssl_op_ignore_unexpected_eof() -> list[str]: + return [ + "SSL_OP_IGNORE_UNEXPECTED_EOF", + ] + + +def cryptography_has_get_extms_support() -> list[str]: + return ["SSL_get_extms_support"] + + +# This is a mapping of +# {condition: function-returning-names-dependent-on-that-condition} so we can +# loop over them and delete unsupported names at runtime. It will be removed +# when cffi supports #if in cdef. We use functions instead of just a dict of +# lists so we can use coverage to measure which are used. +CONDITIONAL_NAMES = { + "Cryptography_HAS_SET_CERT_CB": cryptography_has_set_cert_cb, + "Cryptography_HAS_SSL_ST": cryptography_has_ssl_st, + "Cryptography_HAS_TLS_ST": cryptography_has_tls_st, + "Cryptography_HAS_SIGALGS": cryptography_has_ssl_sigalgs, + "Cryptography_HAS_PSK": cryptography_has_psk, + "Cryptography_HAS_PSK_TLSv1_3": cryptography_has_psk_tlsv13, + "Cryptography_HAS_CUSTOM_EXT": cryptography_has_custom_ext, + "Cryptography_HAS_TLSv1_3_FUNCTIONS": cryptography_has_tlsv13_functions, + "Cryptography_HAS_ENGINE": cryptography_has_engine, + "Cryptography_HAS_VERIFIED_CHAIN": cryptography_has_verified_chain, + "Cryptography_HAS_SRTP": cryptography_has_srtp, + "Cryptography_HAS_OP_NO_RENEGOTIATION": ( + cryptography_has_op_no_renegotiation + ), + "Cryptography_HAS_DTLS_GET_DATA_MTU": cryptography_has_dtls_get_data_mtu, + "Cryptography_HAS_SSL_COOKIE": cryptography_has_ssl_cookie, + "Cryptography_HAS_PRIME_CHECKS": cryptography_has_prime_checks, + "Cryptography_HAS_UNEXPECTED_EOF_WHILE_READING": ( + cryptography_has_unexpected_eof_while_reading + ), + "Cryptography_HAS_SSL_OP_IGNORE_UNEXPECTED_EOF": ( + cryptography_has_ssl_op_ignore_unexpected_eof + ), + "Cryptography_HAS_GET_EXTMS_SUPPORT": cryptography_has_get_extms_support, +} diff --git a/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/binding.py b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/binding.py new file mode 100644 index 0000000000000000000000000000000000000000..d4dfeef485d1cd08ac18093019bb864f18e8ab38 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/cryptography/hazmat/bindings/openssl/binding.py @@ -0,0 +1,121 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import annotations + +import os +import sys +import threading +import types +import typing +import warnings + +import cryptography +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import _openssl, openssl +from cryptography.hazmat.bindings.openssl._conditional import CONDITIONAL_NAMES + + +def _openssl_assert(ok: bool) -> None: + if not ok: + errors = openssl.capture_error_stack() + + raise InternalError( + "Unknown OpenSSL error. This error is commonly encountered when " + "another library is not cleaning up the OpenSSL error stack. If " + "you are using cryptography with another library that uses " + "OpenSSL try disabling it before reporting a bug. Otherwise " + "please file an issue at https://github.com/pyca/cryptography/" + "issues with information on how to reproduce " + f"this. ({errors!r})", + errors, + ) + + +def build_conditional_library( + lib: typing.Any, + conditional_names: dict[str, typing.Callable[[], list[str]]], +) -> typing.Any: + conditional_lib = types.ModuleType("lib") + conditional_lib._original_lib = lib # type: ignore[attr-defined] + excluded_names = set() + for condition, names_cb in conditional_names.items(): + if not getattr(lib, condition): + excluded_names.update(names_cb()) + + for attr in dir(lib): + if attr not in excluded_names: + setattr(conditional_lib, attr, getattr(lib, attr)) + + return conditional_lib + + +class Binding: + """ + OpenSSL API wrapper. + """ + + lib: typing.ClassVar = None + ffi = _openssl.ffi + _lib_loaded = False + _init_lock = threading.Lock() + + def __init__(self) -> None: + self._ensure_ffi_initialized() + + @classmethod + def _ensure_ffi_initialized(cls) -> None: + with cls._init_lock: + if not cls._lib_loaded: + cls.lib = build_conditional_library( + _openssl.lib, CONDITIONAL_NAMES + ) + cls._lib_loaded = True + + @classmethod + def init_static_locks(cls) -> None: + cls._ensure_ffi_initialized() + + +def _verify_package_version(version: str) -> None: + # Occasionally we run into situations where the version of the Python + # package does not match the version of the shared object that is loaded. + # This may occur in environments where multiple versions of cryptography + # are installed and available in the python path. To avoid errors cropping + # up later this code checks that the currently imported package and the + # shared object that were loaded have the same version and raise an + # ImportError if they do not + so_package_version = _openssl.ffi.string( + _openssl.lib.CRYPTOGRAPHY_PACKAGE_VERSION + ) + if version.encode("ascii") != so_package_version: + raise ImportError( + "The version of cryptography does not match the loaded " + "shared object. This can happen if you have multiple copies of " + "cryptography installed in your Python path. Please try creating " + "a new virtual environment to resolve this issue. " + f"Loaded python version: {version}, " + f"shared object version: {so_package_version}" + ) + + _openssl_assert( + _openssl.lib.OpenSSL_version_num() == openssl.openssl_version(), + ) + + +_verify_package_version(cryptography.__version__) + +Binding.init_static_locks() + +if ( + sys.platform == "win32" + and os.environ.get("PROCESSOR_ARCHITEW6432") is not None +): + warnings.warn( + "You are using cryptography on a 32-bit Python on a 64-bit Windows " + "Operating System. Cryptography will be significantly faster if you " + "switch to using a 64-bit Python.", + UserWarning, + stacklevel=2, + ) diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/LICENSE b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..4b114f7cf9ae9407e8871da2f75f3300b57d9d27 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/METADATA @@ -0,0 +1,74 @@ +Metadata-Version: 2.1 +Name: google-api-core +Version: 2.24.0 +Summary: Google API client core library +Author-email: Google LLC +License: Apache 2.0 +Project-URL: Documentation, https://googleapis.dev/python/google-api-core/latest/ +Project-URL: Repository, https://github.com/googleapis/python-api-core +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Operating System :: OS Independent +Classifier: Topic :: Internet +Requires-Python: >=3.7 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: googleapis-common-protos<2.0.dev0,>=1.56.2 +Requires-Dist: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.19.5 +Requires-Dist: proto-plus<2.0.0dev,>=1.22.3 +Requires-Dist: google-auth<3.0.dev0,>=2.14.1 +Requires-Dist: requests<3.0.0.dev0,>=2.18.0 +Requires-Dist: proto-plus<2.0.0dev,>=1.25.0; python_version >= "3.13" +Provides-Extra: async_rest +Requires-Dist: google-auth[aiohttp]<3.0.dev0,>=2.35.0; extra == "async-rest" +Provides-Extra: grpc +Requires-Dist: grpcio<2.0dev,>=1.33.2; extra == "grpc" +Requires-Dist: grpcio-status<2.0.dev0,>=1.33.2; extra == "grpc" +Requires-Dist: grpcio<2.0dev,>=1.49.1; python_version >= "3.11" and extra == "grpc" +Requires-Dist: grpcio-status<2.0.dev0,>=1.49.1; python_version >= "3.11" and extra == "grpc" +Provides-Extra: grpcgcp +Requires-Dist: grpcio-gcp<1.0.dev0,>=0.2.2; extra == "grpcgcp" +Provides-Extra: grpcio-gcp +Requires-Dist: grpcio-gcp<1.0.dev0,>=0.2.2; extra == "grpcio-gcp" + +Core Library for Google Client Libraries +======================================== + +|pypi| |versions| + +This library is not meant to stand-alone. Instead it defines +common helpers used by all Google API clients. For more information, see the +`documentation`_. + +.. |pypi| image:: https://img.shields.io/pypi/v/google-api_core.svg + :target: https://pypi.org/project/google-api_core/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/google-api_core.svg + :target: https://pypi.org/project/google-api_core/ +.. _documentation: https://googleapis.dev/python/google-api-core/latest + + +Supported Python Versions +------------------------- +Python >= 3.7 + + +Unsupported Python Versions +--------------------------- + +Python == 2.7, Python == 3.5, Python == 3.6. + +The last version of this library compatible with Python 2.7 and 3.5 is +`google-api-core==1.31.1`. + +The last version of this library compatible with Python 3.6 is +`google-api-core==2.8.2`. diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..405a94297398be7297f628fcf6392d6335ed36ba --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/RECORD @@ -0,0 +1,125 @@ +google/api_core/__init__.py,sha256=bCgLRZtOkaVlSxTPG_o1x4V0w5FJAWREIlnq3kCfqeY,782 +google/api_core/__pycache__/__init__.cpython-311.pyc,, +google/api_core/__pycache__/_rest_streaming_base.cpython-311.pyc,, +google/api_core/__pycache__/bidi.cpython-311.pyc,, +google/api_core/__pycache__/client_info.cpython-311.pyc,, +google/api_core/__pycache__/client_logging.cpython-311.pyc,, +google/api_core/__pycache__/client_options.cpython-311.pyc,, +google/api_core/__pycache__/datetime_helpers.cpython-311.pyc,, +google/api_core/__pycache__/exceptions.cpython-311.pyc,, +google/api_core/__pycache__/extended_operation.cpython-311.pyc,, +google/api_core/__pycache__/general_helpers.cpython-311.pyc,, +google/api_core/__pycache__/grpc_helpers.cpython-311.pyc,, +google/api_core/__pycache__/grpc_helpers_async.cpython-311.pyc,, +google/api_core/__pycache__/iam.cpython-311.pyc,, +google/api_core/__pycache__/operation.cpython-311.pyc,, +google/api_core/__pycache__/operation_async.cpython-311.pyc,, +google/api_core/__pycache__/page_iterator.cpython-311.pyc,, +google/api_core/__pycache__/page_iterator_async.cpython-311.pyc,, +google/api_core/__pycache__/path_template.cpython-311.pyc,, +google/api_core/__pycache__/protobuf_helpers.cpython-311.pyc,, +google/api_core/__pycache__/rest_helpers.cpython-311.pyc,, +google/api_core/__pycache__/rest_streaming.cpython-311.pyc,, +google/api_core/__pycache__/rest_streaming_async.cpython-311.pyc,, +google/api_core/__pycache__/retry_async.cpython-311.pyc,, +google/api_core/__pycache__/timeout.cpython-311.pyc,, +google/api_core/__pycache__/universe.cpython-311.pyc,, +google/api_core/__pycache__/version.cpython-311.pyc,, +google/api_core/__pycache__/version_header.cpython-311.pyc,, +google/api_core/_rest_streaming_base.py,sha256=AlkPe71v0kRUeWP5yn6N1KbxCxKhr-vfQOCgoF6x8ZE,4351 +google/api_core/bidi.py,sha256=wpC9Lthrh62Jxp9WzRas1W78UiHO5OPuHEvx3SHcvgg,27940 +google/api_core/client_info.py,sha256=WPZIOf-o55wNGogs29KHbYRXOGbPdkMgwYht4aOgqJg,3802 +google/api_core/client_logging.py,sha256=o7VrcpJ5yqIfdpBDGKTIIVaqIfl5Ppr_AxiOfyKIGTk,5023 +google/api_core/client_options.py,sha256=YJ9cvYjVK3Y_j32uIsHCX7ugHwQOLcMy0Y_f9XDL_vI,5818 +google/api_core/datetime_helpers.py,sha256=5gFi7n0r-xVImQdj6rQKNwk58m2LcMF9WliXGHbBsDA,9034 +google/api_core/exceptions.py,sha256=LhvwbDC8QydNUQhSXbfey9JaZnhysDATjk1dRg6vPG0,21135 +google/api_core/extended_operation.py,sha256=r9xSOblNF35lwn2hrrjUQ-f3JDoo0a4Z8xwOy_VkvL0,8632 +google/api_core/future/__init__.py,sha256=7sToxNNu9c_xqcpmO8dbrcSLOOxplnYOOSXjOX9QIXw,702 +google/api_core/future/__pycache__/__init__.cpython-311.pyc,, +google/api_core/future/__pycache__/_helpers.cpython-311.pyc,, +google/api_core/future/__pycache__/async_future.cpython-311.pyc,, +google/api_core/future/__pycache__/base.cpython-311.pyc,, +google/api_core/future/__pycache__/polling.cpython-311.pyc,, +google/api_core/future/_helpers.py,sha256=jA6m2L1aqlOJA-9NdC1BDosPksZQ7FmLLYWDOrsQOPc,1248 +google/api_core/future/async_future.py,sha256=7rOK0tzud8MCoUwO9AjF-3OQDtELwhtp2ONltSB3GEI,5355 +google/api_core/future/base.py,sha256=SHyudamSWR7EyUsYaQ-XrGGkLeYClSfXfsHIHSqDIYI,1763 +google/api_core/future/polling.py,sha256=0HUw1bp7ZLgEqMtwsvxIXNMHQbHgsP6TpmpVrMbjJ2I,14349 +google/api_core/gapic_v1/__init__.py,sha256=r6kCwKznSXPTYRdz4C384fscefaw_rXP2bzJdnzEVnw,988 +google/api_core/gapic_v1/__pycache__/__init__.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/client_info.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/config.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/config_async.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/method.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/method_async.cpython-311.pyc,, +google/api_core/gapic_v1/__pycache__/routing_header.cpython-311.pyc,, +google/api_core/gapic_v1/client_info.py,sha256=91RMt_6UUk0wBLdS-yOPz2FUdnbvM5h1GbvTCG5qlmo,2341 +google/api_core/gapic_v1/config.py,sha256=5isOOYPSZCXpDcJDJiwmTxGTUo0RjxJJvW2yjqBR4BI,6300 +google/api_core/gapic_v1/config_async.py,sha256=_jrB5Yv6rxxSU6KwzOxWQ-G_x5mXilpSFAgnQ_6ktrU,1728 +google/api_core/gapic_v1/method.py,sha256=SnMqRoKKCRph9xpnQvQ29SGjCd9WVpHEPK60X-uPyWM,9494 +google/api_core/gapic_v1/method_async.py,sha256=L8BHV3SkvKTDqVSonDuUY1OIRMPEqfsOsTitYRQ_UwQ,2090 +google/api_core/gapic_v1/routing_header.py,sha256=kJKOYpNS2mgSZa4Qt8Ib2Q5ONfNwpJwbNloVJ8e2wMs,3093 +google/api_core/general_helpers.py,sha256=ZrYwDg7VTgtaQlFk_fCeFTKYZD62JMQdZRhbQhbQL_c,681 +google/api_core/grpc_helpers.py,sha256=r262_0OWGqkECOlydA2dp40KG9Qqhnw72qJ7Pj9hpUU,23291 +google/api_core/grpc_helpers_async.py,sha256=s389VG5fLPOkIlhRuKqnuMW76yeEjKrrvWtS66lNp3o,12224 +google/api_core/iam.py,sha256=BGz63HtOP5_5oH9Zs93RP0Y6Qshty2eOhFEYj_CoE64,13213 +google/api_core/operation.py,sha256=mHWay2vrNbEliv5YWFzyXBywbQdy_VPW98BALh514PA,13198 +google/api_core/operation_async.py,sha256=XdunwVY6aKA-K0OK-5_dYbqjbvF1DLTYUUL4IOztld4,8046 +google/api_core/operations_v1/__init__.py,sha256=ncvxAGOrunbMNRoQ9n1Io1p1nRN_LV5DutV52UidV8k,1638 +google/api_core/operations_v1/__pycache__/__init__.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/abstract_operations_base_client.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/abstract_operations_client.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/operations_async_client.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/operations_client.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/operations_client_config.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/operations_rest_client_async.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/pagers.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/pagers_async.cpython-311.pyc,, +google/api_core/operations_v1/__pycache__/pagers_base.cpython-311.pyc,, +google/api_core/operations_v1/abstract_operations_base_client.py,sha256=JoAlWWxuj_TpbAv9GCBt6_BMhflvIoR-rg9TPSOJ3is,14861 +google/api_core/operations_v1/abstract_operations_client.py,sha256=j_ulCLJpyqGh1SY8z5kss9iYBfOwE_XXCTqwQAKpyeI,16073 +google/api_core/operations_v1/operations_async_client.py,sha256=1BENex2y2ovlCHlXR4v5Cfiqk2o36DBWEzPyCCCudbU,14794 +google/api_core/operations_v1/operations_client.py,sha256=-fmbRv_2L_5cJv70WfybRw9EUyLlHB-wTbC-n0Iq4Fg,15274 +google/api_core/operations_v1/operations_client_config.py,sha256=v7B0FiVc5p9HhnpPY1_3FIomFdA-J-4lilomeoC9SkQ,2285 +google/api_core/operations_v1/operations_rest_client_async.py,sha256=qMYVo08Y0jfSU53dmHSDvO7_UL3x8DzJgpvnwAaTyyE,14616 +google/api_core/operations_v1/pagers.py,sha256=WYXqIGNIMbQX-2OUbiqz3ZXScvI_iOxKjxkN6bTP1YQ,2463 +google/api_core/operations_v1/pagers_async.py,sha256=SdFB-eXtOjZCHWICgViao6txHJqgs7vhsso6HT_npOo,2624 +google/api_core/operations_v1/pagers_base.py,sha256=qlizIpOdU-JVeQIMaPRIBmkcsghDX2FQYz5VH3-l9s0,2652 +google/api_core/operations_v1/transports/__init__.py,sha256=Ng5VDMks97QNfbkhFSRKmvNwUv3_IQmLUszCGTeJYvE,1457 +google/api_core/operations_v1/transports/__pycache__/__init__.cpython-311.pyc,, +google/api_core/operations_v1/transports/__pycache__/base.cpython-311.pyc,, +google/api_core/operations_v1/transports/__pycache__/rest.cpython-311.pyc,, +google/api_core/operations_v1/transports/__pycache__/rest_asyncio.cpython-311.pyc,, +google/api_core/operations_v1/transports/base.py,sha256=7WaH1D28xt8S1d5DNZMG1HbHCZMOB76ITJuzABl2vaM,10635 +google/api_core/operations_v1/transports/rest.py,sha256=v_elA9kWQBA2JW3b7FKDBLfJlFfxyR5DLA_tKrPEDtg,19815 +google/api_core/operations_v1/transports/rest_asyncio.py,sha256=t6ub6RgxKqPfRYO5ahy4l6vVqY2EvIKYuJSiT7TYPNw,24822 +google/api_core/page_iterator.py,sha256=FXMfqbhlVYAEVjpojytYAiUluVNYAVSC41MdfAhHAX4,20330 +google/api_core/page_iterator_async.py,sha256=TbuXorRhP1wcQTD3raBJhWgSJP1JwJO_nCKJphCbVdw,10294 +google/api_core/path_template.py,sha256=Lyqqw8OECuw5O7y9x1BJvfNbYEbmx4lnTGqc6opSyHk,11685 +google/api_core/protobuf_helpers.py,sha256=ct_P2z6iYNvum0FZ5Uj-96qf83Q_99TP1qcGwvlO_9c,12448 +google/api_core/py.typed,sha256=q8dgH9l1moUXiufHBVjqI0MuJy4Be9a3rNH8Zl_sICA,78 +google/api_core/rest_helpers.py,sha256=2DsInZiHv0sLd9dfLIbEL2vDJQIybWgxlkxnNFahPnI,3529 +google/api_core/rest_streaming.py,sha256=AwduJ7tYa0_iBhFEqCY696NVmNGWWCm6g4wnTqoVjS4,2209 +google/api_core/rest_streaming_async.py,sha256=5GuzrfYFHfR22d9guOtXvZ1E-VHCCusJyWKVRxOcFuE,3340 +google/api_core/retry/__init__.py,sha256=WhgtLBQO2oK-AehH_AHbGbfWo1IdG5ahUGrs3aFGw0o,2088 +google/api_core/retry/__pycache__/__init__.cpython-311.pyc,, +google/api_core/retry/__pycache__/retry_base.cpython-311.pyc,, +google/api_core/retry/__pycache__/retry_streaming.cpython-311.pyc,, +google/api_core/retry/__pycache__/retry_streaming_async.cpython-311.pyc,, +google/api_core/retry/__pycache__/retry_unary.cpython-311.pyc,, +google/api_core/retry/__pycache__/retry_unary_async.cpython-311.pyc,, +google/api_core/retry/retry_base.py,sha256=WJtEMphRNm1c3ldnuTjCzVu3Xzm-NwjmZhnULpHGVBM,12433 +google/api_core/retry/retry_streaming.py,sha256=kt5ucutVWGYzIsBugqag2DKBwucBmRtkKUvvlKHQ5ew,10860 +google/api_core/retry/retry_streaming_async.py,sha256=7liXfL9o9X9BvDxwFoHdgcClQJq8A9l_Ir5Z_gN4ZvA,14343 +google/api_core/retry/retry_unary.py,sha256=k_0PGsmFwRe4EPUMRHfzrZatDTFwEp9ucPRZDtKKGm4,13338 +google/api_core/retry/retry_unary_async.py,sha256=9mQgLnDqRNBb0enszfE7suqy-5R_C0PQlV3y-s2q_Hw,9415 +google/api_core/retry_async.py,sha256=_r0ROYeQqdATtRMx-q_6o4bPmqFzPyjr_oV3lfloDSM,1514 +google/api_core/timeout.py,sha256=LnElJPn1XJuU8dNZl0YKxw8wqQbGGV94t5K1NiXOGk0,9685 +google/api_core/universe.py,sha256=k_K5J0I3kKQiM2yEHvxeqAWxXEQZKJ2SfDlMAH-rQ08,2952 +google/api_core/version.py,sha256=M5jVH_jvJ6jR8HkmpH2GSu_0LwmVYd4ir5Hj2wjsCaY,598 +google/api_core/version_header.py,sha256=uEFXosCp8UH7XhznG5GQseTYtWNoJHXRPA557DWsUxA,1046 +google_api_core-2.24.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +google_api_core-2.24.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +google_api_core-2.24.0.dist-info/METADATA,sha256=dl3TjSSJtcFiI5YBQDGXTkiMELLRY9p-SckIgbSbhuM,3026 +google_api_core-2.24.0.dist-info/RECORD,, +google_api_core-2.24.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91 +google_api_core-2.24.0.dist-info/top_level.txt,sha256=_1QvSJIhFAGfxb79D6DhB7SUw2X6T4rwnz_LLrbcD3c,7 diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..9b78c44519d450e4036a1ea44361a2ac7b1ee300 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (75.3.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/top_level.txt b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..cb429113e0f9a73019fd799e8052093fea7f0c8b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/google_api_core-2.24.0.dist-info/top_level.txt @@ -0,0 +1 @@ +google diff --git a/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/METADATA b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..99be2236cdd5991715010d60dcebbd1e39e3866a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/METADATA @@ -0,0 +1,616 @@ +Metadata-Version: 2.3 +Name: httpcore +Version: 1.0.7 +Summary: A minimal low-level HTTP client. +Project-URL: Documentation, https://www.encode.io/httpcore +Project-URL: Homepage, https://www.encode.io/httpcore/ +Project-URL: Source, https://github.com/encode/httpcore +Author-email: Tom Christie +License: BSD-3-Clause +Classifier: Development Status :: 3 - Alpha +Classifier: Environment :: Web Environment +Classifier: Framework :: AsyncIO +Classifier: Framework :: Trio +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.8 +Requires-Dist: certifi +Requires-Dist: h11<0.15,>=0.13 +Provides-Extra: asyncio +Requires-Dist: anyio<5.0,>=4.0; extra == 'asyncio' +Provides-Extra: http2 +Requires-Dist: h2<5,>=3; extra == 'http2' +Provides-Extra: socks +Requires-Dist: socksio==1.*; extra == 'socks' +Provides-Extra: trio +Requires-Dist: trio<1.0,>=0.22.0; extra == 'trio' +Description-Content-Type: text/markdown + +# HTTP Core + +[![Test Suite](https://github.com/encode/httpcore/workflows/Test%20Suite/badge.svg)](https://github.com/encode/httpcore/actions) +[![Package version](https://badge.fury.io/py/httpcore.svg)](https://pypi.org/project/httpcore/) + +> *Do one thing, and do it well.* + +The HTTP Core package provides a minimal low-level HTTP client, which does +one thing only. Sending HTTP requests. + +It does not provide any high level model abstractions over the API, +does not handle redirects, multipart uploads, building authentication headers, +transparent HTTP caching, URL parsing, session cookie handling, +content or charset decoding, handling JSON, environment based configuration +defaults, or any of that Jazz. + +Some things HTTP Core does do: + +* Sending HTTP requests. +* Thread-safe / task-safe connection pooling. +* HTTP(S) proxy & SOCKS proxy support. +* Supports HTTP/1.1 and HTTP/2. +* Provides both sync and async interfaces. +* Async backend support for `asyncio` and `trio`. + +## Requirements + +Python 3.8+ + +## Installation + +For HTTP/1.1 only support, install with: + +```shell +$ pip install httpcore +``` + +There are also a number of optional extras available... + +```shell +$ pip install httpcore['asyncio,trio,http2,socks'] +``` + +## Sending requests + +Send an HTTP request: + +```python +import httpcore + +response = httpcore.request("GET", "https://www.example.com/") + +print(response) +# +print(response.status) +# 200 +print(response.headers) +# [(b'Accept-Ranges', b'bytes'), (b'Age', b'557328'), (b'Cache-Control', b'max-age=604800'), ...] +print(response.content) +# b'\n\n\nExample Domain\n\n\n ...' +``` + +The top-level `httpcore.request()` function is provided for convenience. In practice whenever you're working with `httpcore` you'll want to use the connection pooling functionality that it provides. + +```python +import httpcore + +http = httpcore.ConnectionPool() +response = http.request("GET", "https://www.example.com/") +``` + +Once you're ready to get going, [head over to the documentation](https://www.encode.io/httpcore/). + +## Motivation + +You *probably* don't want to be using HTTP Core directly. It might make sense if +you're writing something like a proxy service in Python, and you just want +something at the lowest possible level, but more typically you'll want to use +a higher level client library, such as `httpx`. + +The motivation for `httpcore` is: + +* To provide a reusable low-level client library, that other packages can then build on top of. +* To provide a *really clear interface split* between the networking code and client logic, + so that each is easier to understand and reason about in isolation. + +## Dependencies + +The `httpcore` package has the following dependencies... + +* `h11` +* `certifi` + +And the following optional extras... + +* `anyio` - Required by `pip install httpcore['asyncio']`. +* `trio` - Required by `pip install httpcore['trio']`. +* `h2` - Required by `pip install httpcore['http2']`. +* `socksio` - Required by `pip install httpcore['socks']`. + +## Versioning + +We use [SEMVER for our versioning policy](https://semver.org/). + +For changes between package versions please see our [project changelog](CHANGELOG.md). + +We recommend pinning your requirements either the most current major version, or a more specific version range: + +```python +pip install 'httpcore==1.*' +``` +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## Version 1.0.7 (November 15th, 2024) + +- Support `proxy=…` configuration on `ConnectionPool()`. (#974) + +## Version 1.0.6 (October 1st, 2024) + +- Relax `trio` dependency pinning. (#956) +- Handle `trio` raising `NotImplementedError` on unsupported platforms. (#955) +- Handle mapping `ssl.SSLError` to `httpcore.ConnectError`. (#918) + +## 1.0.5 (March 27th, 2024) + +- Handle `EndOfStream` exception for anyio backend. (#899) +- Allow trio `0.25.*` series in package dependancies. (#903) + +## 1.0.4 (February 21st, 2024) + +- Add `target` request extension. (#888) +- Fix support for connection `Upgrade` and `CONNECT` when some data in the stream has been read. (#882) + +## 1.0.3 (February 13th, 2024) + +- Fix support for async cancellations. (#880) +- Fix trace extension when used with socks proxy. (#849) +- Fix SSL context for connections using the "wss" scheme (#869) + +## 1.0.2 (November 10th, 2023) + +- Fix `float("inf")` timeouts in `Event.wait` function. (#846) + +## 1.0.1 (November 3rd, 2023) + +- Fix pool timeout to account for the total time spent retrying. (#823) +- Raise a neater RuntimeError when the correct async deps are not installed. (#826) +- Add support for synchronous TLS-in-TLS streams. (#840) + +## 1.0.0 (October 6th, 2023) + +From version 1.0 our async support is now optional, as the package has minimal dependencies by default. + +For async support use either `pip install 'httpcore[asyncio]'` or `pip install 'httpcore[trio]'`. + +The project versioning policy is now explicitly governed by SEMVER. See https://semver.org/. + +- Async support becomes fully optional. (#809) +- Add support for Python 3.12. (#807) + +## 0.18.0 (September 8th, 2023) + +- Add support for HTTPS proxies. (#745, #786) +- Drop Python 3.7 support. (#727) +- Handle `sni_hostname` extension with SOCKS proxy. (#774) +- Handle HTTP/1.1 half-closed connections gracefully. (#641) +- Change the type of `Extensions` from `Mapping[Str, Any]` to `MutableMapping[Str, Any]`. (#762) + +## 0.17.3 (July 5th, 2023) + +- Support async cancellations, ensuring that the connection pool is left in a clean state when cancellations occur. (#726) +- The networking backend interface has [been added to the public API](https://www.encode.io/httpcore/network-backends). Some classes which were previously private implementation detail are now part of the top-level public API. (#699) +- Graceful handling of HTTP/2 GoAway frames, with requests being transparently retried on a new connection. (#730) +- Add exceptions when a synchronous `trace callback` is passed to an asynchronous request or an asynchronous `trace callback` is passed to a synchronous request. (#717) +- Drop Python 3.7 support. (#727) + +## 0.17.2 (May 23th, 2023) + +- Add `socket_options` argument to `ConnectionPool` and `HTTProxy` classes. (#668) +- Improve logging with per-module logger names. (#690) +- Add `sni_hostname` request extension. (#696) +- Resolve race condition during import of `anyio` package. (#692) +- Enable TCP_NODELAY for all synchronous sockets. (#651) + +## 0.17.1 (May 17th, 2023) + +- If 'retries' is set, then allow retries if an SSL handshake error occurs. (#669) +- Improve correctness of tracebacks on network exceptions, by raising properly chained exceptions. (#678) +- Prevent connection-hanging behaviour when HTTP/2 connections are closed by a server-sent 'GoAway' frame. (#679) +- Fix edge-case exception when removing requests from the connection pool. (#680) +- Fix pool timeout edge-case. (#688) + +## 0.17.0 (March 16th, 2023) + +- Add DEBUG level logging. (#648) +- Respect HTTP/2 max concurrent streams when settings updates are sent by server. (#652) +- Increase the allowable HTTP header size to 100kB. (#647) +- Add `retries` option to SOCKS proxy classes. (#643) + +## 0.16.3 (December 20th, 2022) + +- Allow `ws` and `wss` schemes. Allows us to properly support websocket upgrade connections. (#625) +- Forwarding HTTP proxies use a connection-per-remote-host. Required by some proxy implementations. (#637) +- Don't raise `RuntimeError` when closing a connection pool with active connections. Removes some error cases when cancellations are used. (#631) +- Lazy import `anyio`, so that it's no longer a hard dependancy, and isn't imported if unused. (#639) + +## 0.16.2 (November 25th, 2022) + +- Revert 'Fix async cancellation behaviour', which introduced race conditions. (#627) +- Raise `RuntimeError` if attempting to us UNIX domain sockets on Windows. (#619) + +## 0.16.1 (November 17th, 2022) + +- Fix HTTP/1.1 interim informational responses, such as "100 Continue". (#605) + +## 0.16.0 (October 11th, 2022) + +- Support HTTP/1.1 informational responses. (#581) +- Fix async cancellation behaviour. (#580) +- Support `h11` 0.14. (#579) + +## 0.15.0 (May 17th, 2022) + +- Drop Python 3.6 support (#535) +- Ensure HTTP proxy CONNECT requests include `timeout` configuration. (#506) +- Switch to explicit `typing.Optional` for type hints. (#513) +- For `trio` map OSError exceptions to `ConnectError`. (#543) + +## 0.14.7 (February 4th, 2022) + +- Requests which raise a PoolTimeout need to be removed from the pool queue. (#502) +- Fix AttributeError that happened when Socks5Connection were terminated. (#501) + +## 0.14.6 (February 1st, 2022) + +- Fix SOCKS support for `http://` URLs. (#492) +- Resolve race condition around exceptions during streaming a response. (#491) + +## 0.14.5 (January 18th, 2022) + +- SOCKS proxy support. (#478) +- Add proxy_auth argument to HTTPProxy. (#481) +- Improve error message on 'RemoteProtocolError' exception when server disconnects without sending a response. (#479) + +## 0.14.4 (January 5th, 2022) + +- Support HTTP/2 on HTTPS tunnelling proxies. (#468) +- Fix proxy headers missing on HTTP forwarding. (#456) +- Only instantiate SSL context if required. (#457) +- More robust HTTP/2 handling. (#253, #439, #440, #441) + +## 0.14.3 (November 17th, 2021) + +- Fix race condition when removing closed connections from the pool. (#437) + +## 0.14.2 (November 16th, 2021) + +- Failed connections no longer remain in the pool. (Pull #433) + +## 0.14.1 (November 12th, 2021) + +- `max_connections` becomes optional. (Pull #429) +- `certifi` is now included in the install dependancies. (Pull #428) +- `h2` is now strictly optional. (Pull #428) + +## 0.14.0 (November 11th, 2021) + +The 0.14 release is a complete reworking of `httpcore`, comprehensively addressing some underlying issues in the connection pooling, as well as substantially redesigning the API to be more user friendly. + +Some of the lower-level API design also makes the components more easily testable in isolation, and the package now has 100% test coverage. + +See [discussion #419](https://github.com/encode/httpcore/discussions/419) for a little more background. + +There's some other neat bits in there too, such as the "trace" extension, which gives a hook into inspecting the internal events that occur during the request/response cycle. This extension is needed for the HTTPX cli, in order to... + +* Log the point at which the connection is established, and the IP/port on which it is made. +* Determine if the outgoing request should log as HTTP/1.1 or HTTP/2, rather than having to assume it's HTTP/2 if the --http2 flag was passed. (Which may not actually be true.) +* Log SSL version info / certificate info. + +Note that `curio` support is not currently available in 0.14.0. If you're using `httpcore` with `curio` please get in touch, so we can assess if we ought to prioritize it as a feature or not. + +## 0.13.7 (September 13th, 2021) + +- Fix broken error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #403) + +## 0.13.6 (June 15th, 2021) + +### Fixed + +- Close sockets when read or write timeouts occur. (Pull #365) + +## 0.13.5 (June 14th, 2021) + +### Fixed + +- Resolved niggles with AnyIO EOF behaviours. (Pull #358, #362) + +## 0.13.4 (June 9th, 2021) + +### Added + +- Improved error messaging when URL scheme is missing, or a non HTTP(S) scheme is used. (Pull #354) + +### Fixed + +- Switched to `anyio` as the default backend implementation when running with `asyncio`. Resolves some awkward [TLS timeout issues](https://github.com/encode/httpx/discussions/1511). + +## 0.13.3 (May 6th, 2021) + +### Added + +- Support HTTP/2 prior knowledge, using `httpcore.SyncConnectionPool(http1=False)`. (Pull #333) + +### Fixed + +- Handle cases where environment does not provide `select.poll` support. (Pull #331) + +## 0.13.2 (April 29th, 2021) + +### Added + +- Improve error message for specific case of `RemoteProtocolError` where server disconnects without sending a response. (Pull #313) + +## 0.13.1 (April 28th, 2021) + +### Fixed + +- More resiliant testing for closed connections. (Pull #311) +- Don't raise exceptions on ungraceful connection closes. (Pull #310) + +## 0.13.0 (April 21st, 2021) + +The 0.13 release updates the core API in order to match the HTTPX Transport API, +introduced in HTTPX 0.18 onwards. + +An example of making requests with the new interface is: + +```python +with httpcore.SyncConnectionPool() as http: + status_code, headers, stream, extensions = http.handle_request( + method=b'GET', + url=(b'https', b'example.org', 443, b'/'), + headers=[(b'host', b'example.org'), (b'user-agent', b'httpcore')] + stream=httpcore.ByteStream(b''), + extensions={} + ) + body = stream.read() + print(status_code, body) +``` + +### Changed + +- The `.request()` method is now `handle_request()`. (Pull #296) +- The `.arequest()` method is now `.handle_async_request()`. (Pull #296) +- The `headers` argument is no longer optional. (Pull #296) +- The `stream` argument is no longer optional. (Pull #296) +- The `ext` argument is now named `extensions`, and is no longer optional. (Pull #296) +- The `"reason"` extension keyword is now named `"reason_phrase"`. (Pull #296) +- The `"reason_phrase"` and `"http_version"` extensions now use byte strings for their values. (Pull #296) +- The `httpcore.PlainByteStream()` class becomes `httpcore.ByteStream()`. (Pull #296) + +### Added + +- Streams now support a `.read()` interface. (Pull #296) + +### Fixed + +- Task cancellation no longer leaks connections from the connection pool. (Pull #305) + +## 0.12.3 (December 7th, 2020) + +### Fixed + +- Abort SSL connections on close rather than waiting for remote EOF when using `asyncio`. (Pull #167) +- Fix exception raised in case of connect timeouts when using the `anyio` backend. (Pull #236) +- Fix `Host` header precedence for `:authority` in HTTP/2. (Pull #241, #243) +- Handle extra edge case when detecting for socket readability when using `asyncio`. (Pull #242, #244) +- Fix `asyncio` SSL warning when using proxy tunneling. (Pull #249) + +## 0.12.2 (November 20th, 2020) + +### Fixed + +- Properly wrap connect errors on the asyncio backend. (Pull #235) +- Fix `ImportError` occurring on Python 3.9 when using the HTTP/1.1 sync client in a multithreaded context. (Pull #237) + +## 0.12.1 (November 7th, 2020) + +### Added + +- Add connect retries. (Pull #221) + +### Fixed + +- Tweak detection of dropped connections, resolving an issue with open files limits on Linux. (Pull #185) +- Avoid leaking connections when establishing an HTTP tunnel to a proxy has failed. (Pull #223) +- Properly wrap OS errors when using `trio`. (Pull #225) + +## 0.12.0 (October 6th, 2020) + +### Changed + +- HTTP header casing is now preserved, rather than always sent in lowercase. (#216 and python-hyper/h11#104) + +### Added + +- Add Python 3.9 to officially supported versions. + +### Fixed + +- Gracefully handle a stdlib asyncio bug when a connection is closed while it is in a paused-for-reading state. (#201) + +## 0.11.1 (September 28nd, 2020) + +### Fixed + +- Add await to async semaphore release() coroutine (#197) +- Drop incorrect curio classifier (#192) + +## 0.11.0 (September 22nd, 2020) + +The Transport API with 0.11.0 has a couple of significant changes. + +Firstly we've moved changed the request interface in order to allow extensions, which will later enable us to support features +such as trailing headers, HTTP/2 server push, and CONNECT/Upgrade connections. + +The interface changes from: + +```python +def request(method, url, headers, stream, timeout): + return (http_version, status_code, reason, headers, stream) +``` + +To instead including an optional dictionary of extensions on the request and response: + +```python +def request(method, url, headers, stream, ext): + return (status_code, headers, stream, ext) +``` + +Having an open-ended extensions point will allow us to add later support for various optional features, that wouldn't otherwise be supported without these API changes. + +In particular: + +* Trailing headers support. +* HTTP/2 Server Push +* sendfile. +* Exposing raw connection on CONNECT, Upgrade, HTTP/2 bi-di streaming. +* Exposing debug information out of the API, including template name, template context. + +Currently extensions are limited to: + +* request: `timeout` - Optional. Timeout dictionary. +* response: `http_version` - Optional. Include the HTTP version used on the response. +* response: `reason` - Optional. Include the reason phrase used on the response. Only valid with HTTP/1.*. + +See https://github.com/encode/httpx/issues/1274#issuecomment-694884553 for the history behind this. + +Secondly, the async version of `request` is now namespaced as `arequest`. + +This allows concrete transports to support both sync and async implementations on the same class. + +### Added + +- Add curio support. (Pull #168) +- Add anyio support, with `backend="anyio"`. (Pull #169) + +### Changed + +- Update the Transport API to use 'ext' for optional extensions. (Pull #190) +- Update the Transport API to use `.request` and `.arequest` so implementations can support both sync and async. (Pull #189) + +## 0.10.2 (August 20th, 2020) + +### Added + +- Added Unix Domain Socket support. (Pull #139) + +### Fixed + +- Always include the port on proxy CONNECT requests. (Pull #154) +- Fix `max_keepalive_connections` configuration. (Pull #153) +- Fixes behaviour in HTTP/1.1 where server disconnects can be used to signal the end of the response body. (Pull #164) + +## 0.10.1 (August 7th, 2020) + +- Include `max_keepalive_connections` on `AsyncHTTPProxy`/`SyncHTTPProxy` classes. + +## 0.10.0 (August 7th, 2020) + +The most notable change in the 0.10.0 release is that HTTP/2 support is now fully optional. + +Use either `pip install httpcore` for HTTP/1.1 support only, or `pip install httpcore[http2]` for HTTP/1.1 and HTTP/2 support. + +### Added + +- HTTP/2 support becomes optional. (Pull #121, #130) +- Add `local_address=...` support. (Pull #100, #134) +- Add `PlainByteStream`, `IteratorByteStream`, `AsyncIteratorByteStream`. The `AsyncByteSteam` and `SyncByteStream` classes are now pure interface classes. (#133) +- Add `LocalProtocolError`, `RemoteProtocolError` exceptions. (Pull #129) +- Add `UnsupportedProtocol` exception. (Pull #128) +- Add `.get_connection_info()` method. (Pull #102, #137) +- Add better TRACE logs. (Pull #101) + +### Changed + +- `max_keepalive` is deprecated in favour of `max_keepalive_connections`. (Pull #140) + +### Fixed + +- Improve handling of server disconnects. (Pull #112) + +## 0.9.1 (May 27th, 2020) + +### Fixed + +- Proper host resolution for sync case, including IPv6 support. (Pull #97) +- Close outstanding connections when connection pool is closed. (Pull #98) + +## 0.9.0 (May 21th, 2020) + +### Changed + +- URL port becomes an `Optional[int]` instead of `int`. (Pull #92) + +### Fixed + +- Honor HTTP/2 max concurrent streams settings. (Pull #89, #90) +- Remove incorrect debug log. (Pull #83) + +## 0.8.4 (May 11th, 2020) + +### Added + +- Logging via HTTPCORE_LOG_LEVEL and HTTPX_LOG_LEVEL environment variables +and TRACE level logging. (Pull #79) + +### Fixed + +- Reuse of connections on HTTP/2 in close concurrency situations. (Pull #81) + +## 0.8.3 (May 6rd, 2020) + +### Fixed + +- Include `Host` and `Accept` headers on proxy "CONNECT" requests. +- De-duplicate any headers also contained in proxy_headers. +- HTTP/2 flag not being passed down to proxy connections. + +## 0.8.2 (May 3rd, 2020) + +### Fixed + +- Fix connections using proxy forwarding requests not being added to the +connection pool properly. (Pull #70) + +## 0.8.1 (April 30th, 2020) + +### Changed + +- Allow inherintance of both `httpcore.AsyncByteStream`, `httpcore.SyncByteStream` without type conflicts. + +## 0.8.0 (April 30th, 2020) + +### Fixed + +- Fixed tunnel proxy support. + +### Added + +- New `TimeoutException` base class. + +## 0.7.0 (March 5th, 2020) + +- First integration with HTTPX. diff --git a/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/RECORD b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..234f33556e2b584fc36fbe23fe2ff1d77a0620f5 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/RECORD @@ -0,0 +1,68 @@ +httpcore-1.0.7.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +httpcore-1.0.7.dist-info/METADATA,sha256=ATe1rdfnyvJCveGq1xl8q7B27Suta1I2xVcfYU-my4M,21265 +httpcore-1.0.7.dist-info/RECORD,, +httpcore-1.0.7.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87 +httpcore-1.0.7.dist-info/licenses/LICENSE.md,sha256=_ctZFUx0y6uhahEkL3dAvqnyPW_rVUeRfYxflKgDkqU,1518 +httpcore/__init__.py,sha256=LrhuDP3kqwQW-464qRK_Q7B72Zp0LklpkEqbqUHAh2E,3357 +httpcore/__pycache__/__init__.cpython-311.pyc,, +httpcore/__pycache__/_api.cpython-311.pyc,, +httpcore/__pycache__/_exceptions.cpython-311.pyc,, +httpcore/__pycache__/_models.cpython-311.pyc,, +httpcore/__pycache__/_ssl.cpython-311.pyc,, +httpcore/__pycache__/_synchronization.cpython-311.pyc,, +httpcore/__pycache__/_trace.cpython-311.pyc,, +httpcore/__pycache__/_utils.cpython-311.pyc,, +httpcore/_api.py,sha256=unZmeDschBWCGCPCwkS3Wot9euK6bg_kKxLtGTxw214,3146 +httpcore/_async/__init__.py,sha256=EWdl2v4thnAHzJpqjU4h2a8DUiGAvNiWrkii9pfhTf0,1221 +httpcore/_async/__pycache__/__init__.cpython-311.pyc,, +httpcore/_async/__pycache__/connection.cpython-311.pyc,, +httpcore/_async/__pycache__/connection_pool.cpython-311.pyc,, +httpcore/_async/__pycache__/http11.cpython-311.pyc,, +httpcore/_async/__pycache__/http2.cpython-311.pyc,, +httpcore/_async/__pycache__/http_proxy.cpython-311.pyc,, +httpcore/_async/__pycache__/interfaces.cpython-311.pyc,, +httpcore/_async/__pycache__/socks_proxy.cpython-311.pyc,, +httpcore/_async/connection.py,sha256=6OcPXqMEfc0BU38_-iHUNDd1vKSTc2UVT09XqNb_BOk,8449 +httpcore/_async/connection_pool.py,sha256=DOIQ2s2ZCf9qfwxhzMprTPLqCL8OxGXiKF6qRHxvVyY,17307 +httpcore/_async/http11.py,sha256=-qM9bV7PjSQF5vxs37-eUXOIFwbIjPcZbNliuX9TtBw,13880 +httpcore/_async/http2.py,sha256=2mPEUDu8jwx99MVDhDKBu1e8ajCVEkBOu1jUQLk0KR8,23648 +httpcore/_async/http_proxy.py,sha256=2zVkrlv-Ds-rWGaqaXlrhEJiAQFPo23BT3Gq_sWoBXU,14701 +httpcore/_async/interfaces.py,sha256=jTiaWL83pgpGC9ziv90ZfwaKNMmHwmOalzaKiuTxATo,4455 +httpcore/_async/socks_proxy.py,sha256=lLKgLlggPfhFlqi0ODeBkOWvt9CghBBUyqsnsU1tx6Q,13841 +httpcore/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +httpcore/_backends/__pycache__/__init__.cpython-311.pyc,, +httpcore/_backends/__pycache__/anyio.cpython-311.pyc,, +httpcore/_backends/__pycache__/auto.cpython-311.pyc,, +httpcore/_backends/__pycache__/base.cpython-311.pyc,, +httpcore/_backends/__pycache__/mock.cpython-311.pyc,, +httpcore/_backends/__pycache__/sync.cpython-311.pyc,, +httpcore/_backends/__pycache__/trio.cpython-311.pyc,, +httpcore/_backends/anyio.py,sha256=x8PgEhXRC8bVqsdzk_YJx8Y6d9Tub06CuUSwnbmtqoY,5252 +httpcore/_backends/auto.py,sha256=zO136PKZmsaTDK-HRk84eA-MUg8_2wJf4NvmK432Aio,1662 +httpcore/_backends/base.py,sha256=aShgRdZnMmRhFWHetjumlM73f8Kz1YOAyCUP_4kHslA,3042 +httpcore/_backends/mock.py,sha256=er9T436uSe7NLrfiLa4x6Nuqg5ivQ693CxWYCWsgbH4,4077 +httpcore/_backends/sync.py,sha256=bhE4d9iK9Umxdsdsgm2EfKnXaBms2WggGYU-7jmUujU,7977 +httpcore/_backends/trio.py,sha256=LHu4_Mr5MswQmmT3yE4oLgf9b_JJfeVS4BjDxeJc7Ro,5996 +httpcore/_exceptions.py,sha256=looCKga3_YVYu3s-d3L9RMPRJyhsY7fiuuGxvkOD0c0,1184 +httpcore/_models.py,sha256=IO2CcXcdpovRcLTdGFGB6RyBZdEm2h_TOmoCc4rEKho,17623 +httpcore/_ssl.py,sha256=srqmSNU4iOUvWF-SrJvb8G_YEbHFELOXQOwdDIBTS9c,187 +httpcore/_sync/__init__.py,sha256=JBDIgXt5la1LCJ1sLQeKhjKFpLnpNr8Svs6z2ni3fgg,1141 +httpcore/_sync/__pycache__/__init__.cpython-311.pyc,, +httpcore/_sync/__pycache__/connection.cpython-311.pyc,, +httpcore/_sync/__pycache__/connection_pool.cpython-311.pyc,, +httpcore/_sync/__pycache__/http11.cpython-311.pyc,, +httpcore/_sync/__pycache__/http2.cpython-311.pyc,, +httpcore/_sync/__pycache__/http_proxy.cpython-311.pyc,, +httpcore/_sync/__pycache__/interfaces.cpython-311.pyc,, +httpcore/_sync/__pycache__/socks_proxy.cpython-311.pyc,, +httpcore/_sync/connection.py,sha256=9exGOb3PB-Mp2T1-sckSeL2t-tJ_9-NXomV8ihmWCgU,8238 +httpcore/_sync/connection_pool.py,sha256=a-T8LTsUxc7r0Ww1atfHSDoWPjQ0fA8Ul7S3-F0Mj70,16955 +httpcore/_sync/http11.py,sha256=IFobD1Md5JFlJGKWnh1_Q3epikUryI8qo09v8MiJIEA,13476 +httpcore/_sync/http2.py,sha256=IZOBL1nNpOKJYwTSHYWtscD3zjSg8f85-63-o5RedVc,23112 +httpcore/_sync/http_proxy.py,sha256=_al_6crKuEZu2wyvu493RZImJdBJnj5oGKNjLOJL2Zo,14463 +httpcore/_sync/interfaces.py,sha256=snXON42vUDHO5JBJvo8D4VWk2Wat44z2OXXHDrjbl94,4344 +httpcore/_sync/socks_proxy.py,sha256=zegZW9Snqj2_992DFJa8_CppOVBkVL4AgwduRkStakQ,13614 +httpcore/_synchronization.py,sha256=zSi13mAColBnknjZBknUC6hKNDQT4C6ijnezZ-r0T2s,9434 +httpcore/_trace.py,sha256=ck6ZoIzYTkdNAIfq5MGeKqBXDtqjOX-qfYwmZFbrGco,3952 +httpcore/_utils.py,sha256=_RLgXYOAYC350ikALV59GZ68IJrdocRZxPs9PjmzdFY,1537 +httpcore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..21aaa72961a8af71c17d2cb3b76d5f7f567100e4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.26.3 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/licenses/LICENSE.md b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..311b2b56c53f678ab95fc0def708c675d521a807 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore-1.0.7.dist-info/licenses/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2020, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/__init__.py b/.venv/lib/python3.11/site-packages/httpcore/_backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/auto.cpython-311.pyc b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/auto.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed3451678a8e4791112b91f9df2fb31c3c113b94 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/auto.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/base.cpython-311.pyc b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/base.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c4ca4f422d274999597af5fd29b0841f2fd50d59 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/base.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/mock.cpython-311.pyc b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/mock.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..edb18e1ec0bf9bfa37b912c4fd3db5484ade3bc3 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/mock.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/sync.cpython-311.pyc b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/sync.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d290610d7853115e6012a8ac42e50c80c3be8c3d Binary files /dev/null and b/.venv/lib/python3.11/site-packages/httpcore/_backends/__pycache__/sync.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/anyio.py b/.venv/lib/python3.11/site-packages/httpcore/_backends/anyio.py new file mode 100644 index 0000000000000000000000000000000000000000..a140095e1b8de022f321a41c0125e0e5febc0749 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore/_backends/anyio.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import ssl +import typing + +import anyio + +from .._exceptions import ( + ConnectError, + ConnectTimeout, + ReadError, + ReadTimeout, + WriteError, + WriteTimeout, + map_exceptions, +) +from .._utils import is_socket_readable +from .base import SOCKET_OPTION, AsyncNetworkBackend, AsyncNetworkStream + + +class AnyIOStream(AsyncNetworkStream): + def __init__(self, stream: anyio.abc.ByteStream) -> None: + self._stream = stream + + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + exc_map = { + TimeoutError: ReadTimeout, + anyio.BrokenResourceError: ReadError, + anyio.ClosedResourceError: ReadError, + anyio.EndOfStream: ReadError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + try: + return await self._stream.receive(max_bytes=max_bytes) + except anyio.EndOfStream: # pragma: nocover + return b"" + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + if not buffer: + return + + exc_map = { + TimeoutError: WriteTimeout, + anyio.BrokenResourceError: WriteError, + anyio.ClosedResourceError: WriteError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + await self._stream.send(item=buffer) + + async def aclose(self) -> None: + await self._stream.aclose() + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + exc_map = { + TimeoutError: ConnectTimeout, + anyio.BrokenResourceError: ConnectError, + anyio.EndOfStream: ConnectError, + ssl.SSLError: ConnectError, + } + with map_exceptions(exc_map): + try: + with anyio.fail_after(timeout): + ssl_stream = await anyio.streams.tls.TLSStream.wrap( + self._stream, + ssl_context=ssl_context, + hostname=server_hostname, + standard_compatible=False, + server_side=False, + ) + except Exception as exc: # pragma: nocover + await self.aclose() + raise exc + return AnyIOStream(ssl_stream) + + def get_extra_info(self, info: str) -> typing.Any: + if info == "ssl_object": + return self._stream.extra(anyio.streams.tls.TLSAttribute.ssl_object, None) + if info == "client_addr": + return self._stream.extra(anyio.abc.SocketAttribute.local_address, None) + if info == "server_addr": + return self._stream.extra(anyio.abc.SocketAttribute.remote_address, None) + if info == "socket": + return self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + if info == "is_readable": + sock = self._stream.extra(anyio.abc.SocketAttribute.raw_socket, None) + return is_socket_readable(sock) + return None + + +class AnyIOBackend(AsyncNetworkBackend): + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_tcp( + remote_host=host, + remote_port=port, + local_host=local_address, + ) + # By default TCP sockets opened in `asyncio` include TCP_NODELAY. + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: # pragma: nocover + if socket_options is None: + socket_options = [] + exc_map = { + TimeoutError: ConnectTimeout, + OSError: ConnectError, + anyio.BrokenResourceError: ConnectError, + } + with map_exceptions(exc_map): + with anyio.fail_after(timeout): + stream: anyio.abc.ByteStream = await anyio.connect_unix(path) + for option in socket_options: + stream._raw_socket.setsockopt(*option) # type: ignore[attr-defined] # pragma: no cover + return AnyIOStream(stream) + + async def sleep(self, seconds: float) -> None: + await anyio.sleep(seconds) # pragma: nocover diff --git a/.venv/lib/python3.11/site-packages/httpcore/_backends/base.py b/.venv/lib/python3.11/site-packages/httpcore/_backends/base.py new file mode 100644 index 0000000000000000000000000000000000000000..cf55c8b10eb543872550be863206fe2f760d0d8d --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore/_backends/base.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import ssl +import time +import typing + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + + +class NetworkStream: + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + def close(self) -> None: + raise NotImplementedError() # pragma: nocover + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class NetworkBackend: + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> NetworkStream: + raise NotImplementedError() # pragma: nocover + + def sleep(self, seconds: float) -> None: + time.sleep(seconds) # pragma: nocover + + +class AsyncNetworkStream: + async def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + raise NotImplementedError() # pragma: nocover + + async def write(self, buffer: bytes, timeout: float | None = None) -> None: + raise NotImplementedError() # pragma: nocover + + async def aclose(self) -> None: + raise NotImplementedError() # pragma: nocover + + async def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + def get_extra_info(self, info: str) -> typing.Any: + return None # pragma: nocover + + +class AsyncNetworkBackend: + async def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> AsyncNetworkStream: + raise NotImplementedError() # pragma: nocover + + async def sleep(self, seconds: float) -> None: + raise NotImplementedError() # pragma: nocover diff --git a/.venv/lib/python3.11/site-packages/httpcore/_ssl.py b/.venv/lib/python3.11/site-packages/httpcore/_ssl.py new file mode 100644 index 0000000000000000000000000000000000000000..c99c5a67945b8a3a3544d481e979c791ab45fe23 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore/_ssl.py @@ -0,0 +1,9 @@ +import ssl + +import certifi + + +def default_ssl_context() -> ssl.SSLContext: + context = ssl.create_default_context() + context.load_verify_locations(certifi.where()) + return context diff --git a/.venv/lib/python3.11/site-packages/httpcore/_synchronization.py b/.venv/lib/python3.11/site-packages/httpcore/_synchronization.py new file mode 100644 index 0000000000000000000000000000000000000000..2ecc9e9c363e2f16c4f934cf41cf871826d6a495 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/httpcore/_synchronization.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import threading +import types + +from ._exceptions import ExceptionMapping, PoolTimeout, map_exceptions + +# Our async synchronization primatives use either 'anyio' or 'trio' depending +# on if they're running under asyncio or trio. + +try: + import trio +except (ImportError, NotImplementedError): # pragma: nocover + trio = None # type: ignore + +try: + import anyio +except ImportError: # pragma: nocover + anyio = None # type: ignore + + +def current_async_library() -> str: + # Determine if we're running under trio or asyncio. + # See https://sniffio.readthedocs.io/en/latest/ + try: + import sniffio + except ImportError: # pragma: nocover + environment = "asyncio" + else: + environment = sniffio.current_async_library() + + if environment not in ("asyncio", "trio"): # pragma: nocover + raise RuntimeError("Running under an unsupported async environment.") + + if environment == "asyncio" and anyio is None: # pragma: nocover + raise RuntimeError( + "Running with asyncio requires installation of 'httpcore[asyncio]'." + ) + + if environment == "trio" and trio is None: # pragma: nocover + raise RuntimeError( + "Running with trio requires installation of 'httpcore[trio]'." + ) + + return environment + + +class AsyncLock: + """ + This is a standard lock. + + In the sync case `Lock` provides thread locking. + In the async case `AsyncLock` provides async locking. + """ + + def __init__(self) -> None: + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a lock with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_lock = trio.Lock() + elif self._backend == "asyncio": + self._anyio_lock = anyio.Lock() + + async def __aenter__(self) -> AsyncLock: + if not self._backend: + self.setup() + + if self._backend == "trio": + await self._trio_lock.acquire() + elif self._backend == "asyncio": + await self._anyio_lock.acquire() + + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self._backend == "trio": + self._trio_lock.release() + elif self._backend == "asyncio": + self._anyio_lock.release() + + +class AsyncThreadLock: + """ + This is a threading-only lock for no-I/O contexts. + + In the sync case `ThreadLock` provides thread locking. + In the async case `AsyncThreadLock` is a no-op. + """ + + def __enter__(self) -> AsyncThreadLock: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + pass + + +class AsyncEvent: + def __init__(self) -> None: + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a lock with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_event = trio.Event() + elif self._backend == "asyncio": + self._anyio_event = anyio.Event() + + def set(self) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + self._trio_event.set() + elif self._backend == "asyncio": + self._anyio_event.set() + + async def wait(self, timeout: float | None = None) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + trio_exc_map: ExceptionMapping = {trio.TooSlowError: PoolTimeout} + timeout_or_inf = float("inf") if timeout is None else timeout + with map_exceptions(trio_exc_map): + with trio.fail_after(timeout_or_inf): + await self._trio_event.wait() + elif self._backend == "asyncio": + anyio_exc_map: ExceptionMapping = {TimeoutError: PoolTimeout} + with map_exceptions(anyio_exc_map): + with anyio.fail_after(timeout): + await self._anyio_event.wait() + + +class AsyncSemaphore: + def __init__(self, bound: int) -> None: + self._bound = bound + self._backend = "" + + def setup(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a semaphore with the correct implementation. + """ + self._backend = current_async_library() + if self._backend == "trio": + self._trio_semaphore = trio.Semaphore( + initial_value=self._bound, max_value=self._bound + ) + elif self._backend == "asyncio": + self._anyio_semaphore = anyio.Semaphore( + initial_value=self._bound, max_value=self._bound + ) + + async def acquire(self) -> None: + if not self._backend: + self.setup() + + if self._backend == "trio": + await self._trio_semaphore.acquire() + elif self._backend == "asyncio": + await self._anyio_semaphore.acquire() + + async def release(self) -> None: + if self._backend == "trio": + self._trio_semaphore.release() + elif self._backend == "asyncio": + self._anyio_semaphore.release() + + +class AsyncShieldCancellation: + # For certain portions of our codebase where we're dealing with + # closing connections during exception handling we want to shield + # the operation from being cancelled. + # + # with AsyncShieldCancellation(): + # ... # clean-up operations, shielded from cancellation. + + def __init__(self) -> None: + """ + Detect if we're running under 'asyncio' or 'trio' and create + a shielded scope with the correct implementation. + """ + self._backend = current_async_library() + + if self._backend == "trio": + self._trio_shield = trio.CancelScope(shield=True) + elif self._backend == "asyncio": + self._anyio_shield = anyio.CancelScope(shield=True) + + def __enter__(self) -> AsyncShieldCancellation: + if self._backend == "trio": + self._trio_shield.__enter__() + elif self._backend == "asyncio": + self._anyio_shield.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + if self._backend == "trio": + self._trio_shield.__exit__(exc_type, exc_value, traceback) + elif self._backend == "asyncio": + self._anyio_shield.__exit__(exc_type, exc_value, traceback) + + +# Our thread-based synchronization primitives... + + +class Lock: + """ + This is a standard lock. + + In the sync case `Lock` provides thread locking. + In the async case `AsyncLock` provides async locking. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + + def __enter__(self) -> Lock: + self._lock.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self._lock.release() + + +class ThreadLock: + """ + This is a threading-only lock for no-I/O contexts. + + In the sync case `ThreadLock` provides thread locking. + In the async case `AsyncThreadLock` is a no-op. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + + def __enter__(self) -> ThreadLock: + self._lock.acquire() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + self._lock.release() + + +class Event: + def __init__(self) -> None: + self._event = threading.Event() + + def set(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> None: + if timeout == float("inf"): # pragma: no cover + timeout = None + if not self._event.wait(timeout=timeout): + raise PoolTimeout() # pragma: nocover + + +class Semaphore: + def __init__(self, bound: int) -> None: + self._semaphore = threading.Semaphore(value=bound) + + def acquire(self) -> None: + self._semaphore.acquire() + + def release(self) -> None: + self._semaphore.release() + + +class ShieldCancellation: + # Thread-synchronous codebases don't support cancellation semantics. + # We have this class because we need to mirror the async and sync + # cases within our package, but it's just a no-op. + def __enter__(self) -> ShieldCancellation: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: types.TracebackType | None = None, + ) -> None: + pass diff --git a/.venv/lib/python3.11/site-packages/httpcore/py.typed b/.venv/lib/python3.11/site-packages/httpcore/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/LICENSE.md b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..19b6b45242c16a1025465309eec2ca5009319de3 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/LICENSE.md @@ -0,0 +1,31 @@ +BSD 3-Clause License + +Copyright (c) 2013-2024, Kim Davies and contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/METADATA b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c42623e9423c23b555d9d352bc5dab518ede02c2 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/METADATA @@ -0,0 +1,250 @@ +Metadata-Version: 2.1 +Name: idna +Version: 3.10 +Summary: Internationalized Domain Names in Applications (IDNA) +Author-email: Kim Davies +Requires-Python: >=3.6 +Description-Content-Type: text/x-rst +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: Name Service (DNS) +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Utilities +Requires-Dist: ruff >= 0.6.2 ; extra == "all" +Requires-Dist: mypy >= 1.11.2 ; extra == "all" +Requires-Dist: pytest >= 8.3.2 ; extra == "all" +Requires-Dist: flake8 >= 7.1.1 ; extra == "all" +Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.rst +Project-URL: Issue tracker, https://github.com/kjd/idna/issues +Project-URL: Source, https://github.com/kjd/idna +Provides-Extra: all + +Internationalized Domain Names in Applications (IDNA) +===================================================== + +Support for the Internationalized Domain Names in +Applications (IDNA) protocol as specified in `RFC 5891 +`_. This is the latest version of +the protocol and is sometimes referred to as “IDNA 2008”. + +This library also provides support for Unicode Technical +Standard 46, `Unicode IDNA Compatibility Processing +`_. + +This acts as a suitable replacement for the “encodings.idna” +module that comes with the Python standard library, but which +only supports the older superseded IDNA specification (`RFC 3490 +`_). + +Basic functions are simply executed: + +.. code-block:: pycon + + >>> import idna + >>> idna.encode('ドメイン.テスト') + b'xn--eckwd4c7c.xn--zckzah' + >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) + ドメイン.テスト + + +Installation +------------ + +This package is available for installation from PyPI: + +.. code-block:: bash + + $ python3 -m pip install idna + + +Usage +----- + +For typical usage, the ``encode`` and ``decode`` functions will take a +domain name argument and perform a conversion to A-labels or U-labels +respectively. + +.. code-block:: pycon + + >>> import idna + >>> idna.encode('ドメイン.テスト') + b'xn--eckwd4c7c.xn--zckzah' + >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) + ドメイン.テスト + +You may use the codec encoding and decoding methods using the +``idna.codec`` module: + +.. code-block:: pycon + + >>> import idna.codec + >>> print('домен.испытание'.encode('idna2008')) + b'xn--d1acufc.xn--80akhbyknj4f' + >>> print(b'xn--d1acufc.xn--80akhbyknj4f'.decode('idna2008')) + домен.испытание + +Conversions can be applied at a per-label basis using the ``ulabel`` or +``alabel`` functions if necessary: + +.. code-block:: pycon + + >>> idna.alabel('测试') + b'xn--0zwm56d' + +Compatibility Mapping (UTS #46) ++++++++++++++++++++++++++++++++ + +As described in `RFC 5895 `_, the +IDNA specification does not normalize input from different potential +ways a user may input a domain name. This functionality, known as +a “mapping”, is considered by the specification to be a local +user-interface issue distinct from IDNA conversion functionality. + +This library provides one such mapping that was developed by the +Unicode Consortium. Known as `Unicode IDNA Compatibility Processing +`_, it provides for both a regular +mapping for typical applications, as well as a transitional mapping to +help migrate from older IDNA 2003 applications. Strings are +preprocessed according to Section 4.4 “Preprocessing for IDNA2008” +prior to the IDNA operations. + +For example, “Königsgäßchen” is not a permissible label as *LATIN +CAPITAL LETTER K* is not allowed (nor are capital letters in general). +UTS 46 will convert this into lower case prior to applying the IDNA +conversion. + +.. code-block:: pycon + + >>> import idna + >>> idna.encode('Königsgäßchen') + ... + idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed + >>> idna.encode('Königsgäßchen', uts46=True) + b'xn--knigsgchen-b4a3dun' + >>> print(idna.decode('xn--knigsgchen-b4a3dun')) + königsgäßchen + +Transitional processing provides conversions to help transition from +the older 2003 standard to the current standard. For example, in the +original IDNA specification, the *LATIN SMALL LETTER SHARP S* (ß) was +converted into two *LATIN SMALL LETTER S* (ss), whereas in the current +IDNA specification this conversion is not performed. + +.. code-block:: pycon + + >>> idna.encode('Königsgäßchen', uts46=True, transitional=True) + 'xn--knigsgsschen-lcb0w' + +Implementers should use transitional processing with caution, only in +rare cases where conversion from legacy labels to current labels must be +performed (i.e. IDNA implementations that pre-date 2008). For typical +applications that just need to convert labels, transitional processing +is unlikely to be beneficial and could produce unexpected incompatible +results. + +``encodings.idna`` Compatibility +++++++++++++++++++++++++++++++++ + +Function calls from the Python built-in ``encodings.idna`` module are +mapped to their IDNA 2008 equivalents using the ``idna.compat`` module. +Simply substitute the ``import`` clause in your code to refer to the new +module name. + +Exceptions +---------- + +All errors raised during the conversion following the specification +should raise an exception derived from the ``idna.IDNAError`` base +class. + +More specific exceptions that may be generated as ``idna.IDNABidiError`` +when the error reflects an illegal combination of left-to-right and +right-to-left characters in a label; ``idna.InvalidCodepoint`` when +a specific codepoint is an illegal character in an IDN label (i.e. +INVALID); and ``idna.InvalidCodepointContext`` when the codepoint is +illegal based on its positional context (i.e. it is CONTEXTO or CONTEXTJ +but the contextual requirements are not satisfied.) + +Building and Diagnostics +------------------------ + +The IDNA and UTS 46 functionality relies upon pre-calculated lookup +tables for performance. These tables are derived from computing against +eligibility criteria in the respective standards. These tables are +computed using the command-line script ``tools/idna-data``. + +This tool will fetch relevant codepoint data from the Unicode repository +and perform the required calculations to identify eligibility. There are +three main modes: + +* ``idna-data make-libdata``. Generates ``idnadata.py`` and + ``uts46data.py``, the pre-calculated lookup tables used for IDNA and + UTS 46 conversions. Implementers who wish to track this library against + a different Unicode version may use this tool to manually generate a + different version of the ``idnadata.py`` and ``uts46data.py`` files. + +* ``idna-data make-table``. Generate a table of the IDNA disposition + (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix + B.1 of RFC 5892 and the pre-computed tables published by `IANA + `_. + +* ``idna-data U+0061``. Prints debugging output on the various + properties associated with an individual Unicode codepoint (in this + case, U+0061), that are used to assess the IDNA and UTS 46 status of a + codepoint. This is helpful in debugging or analysis. + +The tool accepts a number of arguments, described using ``idna-data +-h``. Most notably, the ``--version`` argument allows the specification +of the version of Unicode to be used in computing the table data. For +example, ``idna-data --version 9.0.0 make-libdata`` will generate +library data against Unicode 9.0.0. + + +Additional Notes +---------------- + +* **Packages**. The latest tagged release version is published in the + `Python Package Index `_. + +* **Version support**. This library supports Python 3.6 and higher. + As this library serves as a low-level toolkit for a variety of + applications, many of which strive for broad compatibility with older + Python versions, there is no rush to remove older interpreter support. + Removing support for older versions should be well justified in that the + maintenance burden has become too high. + +* **Python 2**. Python 2 is supported by version 2.x of this library. + Use "idna<3" in your requirements file if you need this library for + a Python 2 application. Be advised that these versions are no longer + actively developed. + +* **Testing**. The library has a test suite based on each rule of the + IDNA specification, as well as tests that are provided as part of the + Unicode Technical Standard 46, `Unicode IDNA Compatibility Processing + `_. + +* **Emoji**. It is an occasional request to support emoji domains in + this library. Encoding of symbols like emoji is expressly prohibited by + the technical standard IDNA 2008 and emoji domains are broadly phased + out across the domain industry due to associated security risks. For + now, applications that need to support these non-compliant labels + may wish to consider trying the encode/decode operation in this library + first, and then falling back to using `encodings.idna`. See `the Github + project `_ for more discussion. + diff --git a/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/RECORD b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..4a47b68eb0234668716c45950639497f8be97a99 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/RECORD @@ -0,0 +1,22 @@ +idna-3.10.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +idna-3.10.dist-info/LICENSE.md,sha256=pZ8LDvNjWHQQmkRhykT_enDVBpboFHZ7-vch1Mmw2w8,1541 +idna-3.10.dist-info/METADATA,sha256=URR5ZyDfQ1PCEGhkYoojqfi2Ra0tau2--lhwG4XSfjI,10158 +idna-3.10.dist-info/RECORD,, +idna-3.10.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81 +idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868 +idna/__pycache__/__init__.cpython-311.pyc,, +idna/__pycache__/codec.cpython-311.pyc,, +idna/__pycache__/compat.cpython-311.pyc,, +idna/__pycache__/core.cpython-311.pyc,, +idna/__pycache__/idnadata.cpython-311.pyc,, +idna/__pycache__/intranges.cpython-311.pyc,, +idna/__pycache__/package_data.cpython-311.pyc,, +idna/__pycache__/uts46data.cpython-311.pyc,, +idna/codec.py,sha256=PEew3ItwzjW4hymbasnty2N2OXvNcgHB-JjrBuxHPYY,3422 +idna/compat.py,sha256=RzLy6QQCdl9784aFhb2EX9EKGCJjg0P3PilGdeXXcx8,316 +idna/core.py,sha256=YJYyAMnwiQEPjVC4-Fqu_p4CJ6yKKuDGmppBNQNQpFs,13239 +idna/idnadata.py,sha256=W30GcIGvtOWYwAjZj4ZjuouUutC6ffgNuyjJy7fZ-lo,78306 +idna/intranges.py,sha256=amUtkdhYcQG8Zr-CoMM_kVRacxkivC1WgxN1b63KKdU,1898 +idna/package_data.py,sha256=q59S3OXsc5VI8j6vSD0sGBMyk6zZ4vWFREE88yCJYKs,21 +idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +idna/uts46data.py,sha256=rt90K9J40gUSwppDPCrhjgi5AA6pWM65dEGRSf6rIhM,239289 diff --git a/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..3b5e64b5e6c4a210201d1676a891fd57b15cda99 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/idna-3.10.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: flit 3.9.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__init__.py b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6ca9a5ef53483c11f847057e60849bf3707bde29 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__init__.py @@ -0,0 +1,5 @@ +from .instrumentation import PrometheusFastApiInstrumentator + +__version__ = "7.0.2" + +Instrumentator = PrometheusFastApiInstrumentator diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84d079c2e2a47e241e809c8c171efa41c33e4e4a Binary files /dev/null and b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/instrumentation.cpython-311.pyc b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/instrumentation.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4afc4f72f97fafdc08a15cb45cd2dd7521922ba4 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/instrumentation.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/metrics.cpython-311.pyc b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/metrics.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb4fe3b796427e7300cb8c5d07d241a6ba2b2234 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/metrics.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/middleware.cpython-311.pyc b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/middleware.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5494cdd7619ab4a3eec7e7354cb58219c0740839 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/middleware.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/routing.cpython-311.pyc b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/routing.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..490b6f118837df824a1ba5097005e6857c5066f9 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/__pycache__/routing.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/instrumentation.py b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/instrumentation.py new file mode 100644 index 0000000000000000000000000000000000000000..13cdadcc6d298fe41e6638aa46d02ab9f8c48f1a --- /dev/null +++ b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/instrumentation.py @@ -0,0 +1,344 @@ +import asyncio +import gzip +import importlib.util +import os +import re +import warnings +from enum import Enum +from typing import Any, Awaitable, Callable, List, Optional, Sequence, Union, cast + +from prometheus_client import ( + CONTENT_TYPE_LATEST, + REGISTRY, + CollectorRegistry, + generate_latest, + multiprocess, +) +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import Response + +from prometheus_fastapi_instrumentator import metrics +from prometheus_fastapi_instrumentator.middleware import ( + PrometheusInstrumentatorMiddleware, +) + + +class PrometheusFastApiInstrumentator: + def __init__( + self, + should_group_status_codes: bool = True, + should_ignore_untemplated: bool = False, + should_group_untemplated: bool = True, + should_round_latency_decimals: bool = False, + should_respect_env_var: bool = False, + should_instrument_requests_inprogress: bool = False, + should_exclude_streaming_duration: bool = False, + excluded_handlers: List[str] = [], + body_handlers: List[str] = [], + round_latency_decimals: int = 4, + env_var_name: str = "ENABLE_METRICS", + inprogress_name: str = "http_requests_inprogress", + inprogress_labels: bool = False, + registry: Union[CollectorRegistry, None] = None, + ) -> None: + """Create a Prometheus FastAPI (and Starlette) Instrumentator. + + Args: + should_group_status_codes (bool): Should status codes be grouped into + `2xx`, `3xx` and so on? Defaults to `True`. + + should_ignore_untemplated (bool): Should requests without a matching + template be ignored? Defaults to `False`. This means that by + default a request like `curl -X GET localhost:80/doesnotexist` + will be ignored. + + should_group_untemplated (bool): Should requests without a matching + template be grouped to handler `none`? Defaults to `True`. + + should_round_latency_decimals: Should recorded latencies be + rounded to a certain number of decimals? + + should_respect_env_var (bool): Should the instrumentator only work - for + example the methods `instrument()` and `expose()` - if a + certain environment variable is set to `true`? Usecase: A base + FastAPI app that is used by multiple distinct apps. The apps + only have to set the variable to be instrumented. Defaults to + `False`. + + should_instrument_requests_inprogress (bool): Enables a gauge that shows + the inprogress requests. See also the related args starting + with `inprogress`. Defaults to `False`. + + should_exclude_streaming_duration: Should the streaming duration be + excluded? Only relevant if default metrics are used. Defaults + to `False`. + + excluded_handlers (List[str]): List of strings that will be compiled + to regex patterns. All matches will be skipped and not + instrumented. Defaults to `[]`. + + body_handlers (List[str]): List of strings that will be compiled + to regex patterns to match handlers for the middleware to + pass through response bodies to instrumentations. So only + relevant for instrumentations that access `info.response.body`. + Note that this has a noticeable negative impact on performance + with responses larger than a few MBs. Defaults to `[]`. + + round_latency_decimals (int): Number of decimals latencies should be + rounded to. Ignored unless `should_round_latency_decimals` is + `True`. Defaults to `4`. + + env_var_name (str): Any valid os environment variable name that will + be checked for existence before instrumentation. Ignored unless + `should_respect_env_var` is `True`. Defaults to `"ENABLE_METRICS"`. + + inprogress_name (str): Name of the gauge. Defaults to + `http_requests_inprogress`. Ignored unless + `should_instrument_requests_inprogress` is `True`. + + inprogress_labels (bool): Should labels `method` and `handler` be + part of the inprogress label? Ignored unless + `should_instrument_requests_inprogress` is `True`. Defaults to `False`. + + registry (CollectorRegistry): A custom Prometheus registry to use. If not + provided, the default `REGISTRY` will be used. This can be useful if + you need to run multiple apps at the same time, with their own + registries, for example during testing. + + Raises: + ValueError: If `PROMETHEUS_MULTIPROC_DIR` env var is found but + doesn't point to a valid directory. + """ + + self.should_group_status_codes = should_group_status_codes + self.should_ignore_untemplated = should_ignore_untemplated + self.should_group_untemplated = should_group_untemplated + self.should_round_latency_decimals = should_round_latency_decimals + self.should_respect_env_var = should_respect_env_var + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.should_exclude_streaming_duration = should_exclude_streaming_duration + + self.round_latency_decimals = round_latency_decimals + self.env_var_name = env_var_name + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + + self.excluded_handlers = [re.compile(path) for path in excluded_handlers] + self.body_handlers = [re.compile(path) for path in body_handlers] + + self.instrumentations: List[Callable[[metrics.Info], None]] = [] + self.async_instrumentations: List[Callable[[metrics.Info], Awaitable[None]]] = [] + + if ( + "prometheus_multiproc_dir" in os.environ + and "PROMETHEUS_MULTIPROC_DIR" not in os.environ + ): + os.environ["PROMETHEUS_MULTIPROC_DIR"] = os.environ[ + "prometheus_multiproc_dir" + ] + warnings.warn( + "prometheus_multiproc_dir variable has been deprecated in favor of the upper case naming PROMETHEUS_MULTIPROC_DIR", + DeprecationWarning, + ) + + if registry: + self.registry = registry + else: + self.registry = REGISTRY + + if "PROMETHEUS_MULTIPROC_DIR" in os.environ: + pmd = os.environ["PROMETHEUS_MULTIPROC_DIR"] + if not os.path.isdir(pmd): + raise ValueError( + f"Env var PROMETHEUS_MULTIPROC_DIR='{pmd}' not a directory." + ) + + def instrument( + self, + app: Starlette, + metric_namespace: str = "", + metric_subsystem: str = "", + should_only_respect_2xx_for_highr: bool = False, + latency_highr_buckets: Sequence[Union[float, str]] = ( + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.25, + 0.5, + 0.75, + 1, + 1.5, + 2, + 2.5, + 3, + 3.5, + 4, + 4.5, + 5, + 7.5, + 10, + 30, + 60, + ), + latency_lowr_buckets: Sequence[Union[float, str]] = (0.1, 0.5, 1), + ) -> "PrometheusFastApiInstrumentator": + """Performs the instrumentation by adding middleware. + + The middleware iterates through all `instrumentations` and executes them. + + Args: + app: Starlette app instance. Note that every FastAPI app is a + Starlette app. + + Raises: + e: Only raised if app itself throws an exception. + + Returns: + self: Instrumentator. Builder Pattern. + """ + + if self.should_respect_env_var and not self._should_instrumentate(): + return self + + app.add_middleware( + PrometheusInstrumentatorMiddleware, + should_group_status_codes=self.should_group_status_codes, + should_ignore_untemplated=self.should_ignore_untemplated, + should_group_untemplated=self.should_group_untemplated, + should_round_latency_decimals=self.should_round_latency_decimals, + should_respect_env_var=self.should_respect_env_var, + should_instrument_requests_inprogress=self.should_instrument_requests_inprogress, + should_exclude_streaming_duration=self.should_exclude_streaming_duration, + round_latency_decimals=self.round_latency_decimals, + env_var_name=self.env_var_name, + inprogress_name=self.inprogress_name, + inprogress_labels=self.inprogress_labels, + instrumentations=self.instrumentations, + async_instrumentations=self.async_instrumentations, + excluded_handlers=self.excluded_handlers, + body_handlers=self.body_handlers, + metric_namespace=metric_namespace, + metric_subsystem=metric_subsystem, + should_only_respect_2xx_for_highr=should_only_respect_2xx_for_highr, + latency_highr_buckets=latency_highr_buckets, + latency_lowr_buckets=latency_lowr_buckets, + registry=self.registry, + ) + return self + + def expose( + self, + app: Starlette, + should_gzip: bool = False, + endpoint: str = "/metrics", + include_in_schema: bool = True, + tags: Optional[List[Union[str, Enum]]] = None, + **kwargs: Any, + ) -> "PrometheusFastApiInstrumentator": + """Exposes endpoint for metrics. + + Args: + app: App instance. Endpoint will be added to this app. This can be + a Starlette app or a FastAPI app. If it is a Starlette app, `tags` + `kwargs` will be ignored. + + should_gzip: Should the endpoint return compressed data? It will + also check for `gzip` in the `Accept-Encoding` header. + Compression consumes more CPU cycles. In most cases it's best + to just leave this option off since network bandwidth is usually + cheaper than CPU cycles. Defaults to `False`. + + endpoint: Endpoint on which metrics should be exposed. + + include_in_schema: Should the endpoint show up in the documentation? + + tags (List[str], optional): If you manage your routes with tags. + Defaults to None. Only passed to FastAPI app. + + kwargs: Will be passed to app. Only passed to FastAPI app. + + Returns: + self: Instrumentator. Builder Pattern. + """ + + if self.should_respect_env_var and not self._should_instrumentate(): + return self + + def metrics(request: Request) -> Response: + """Endpoint that serves Prometheus metrics.""" + + ephemeral_registry = self.registry + if "PROMETHEUS_MULTIPROC_DIR" in os.environ: + ephemeral_registry = CollectorRegistry() + multiprocess.MultiProcessCollector(ephemeral_registry) + + if should_gzip and "gzip" in request.headers.get("Accept-Encoding", ""): + resp = Response( + content=gzip.compress(generate_latest(ephemeral_registry)) + ) + resp.headers["Content-Type"] = CONTENT_TYPE_LATEST + resp.headers["Content-Encoding"] = "gzip" + else: + resp = Response(content=generate_latest(ephemeral_registry)) + resp.headers["Content-Type"] = CONTENT_TYPE_LATEST + + return resp + + route_configured = False + if importlib.util.find_spec("fastapi"): + from fastapi import FastAPI + + if isinstance(app, FastAPI): + fastapi_app: FastAPI = app + fastapi_app.get( + endpoint, include_in_schema=include_in_schema, tags=tags, **kwargs + )(metrics) + route_configured = True + if not route_configured: + app.add_route( + path=endpoint, route=metrics, include_in_schema=include_in_schema + ) + + return self + + def add( + self, + *instrumentation_function: Optional[ + Callable[[metrics.Info], Union[None, Awaitable[None]]] + ], + ) -> "PrometheusFastApiInstrumentator": + """Adds function to list of instrumentations. + + Args: + instrumentation_function: Function + that will be executed during every request handler call (if + not excluded). See above for detailed information on the + interface of the function. + + Returns: + self: Instrumentator. Builder Pattern. + """ + + for func in instrumentation_function: + if func: + if asyncio.iscoroutinefunction(func): + self.async_instrumentations.append( + cast( + Callable[[metrics.Info], Awaitable[None]], + func, + ) + ) + else: + self.instrumentations.append( + cast(Callable[[metrics.Info], None], func) + ) + + return self + + def _should_instrumentate(self) -> bool: + """Checks if instrumentation should be performed based on env var.""" + + return os.getenv(self.env_var_name, "False").lower() in ["true", "1"] diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/metrics.py b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..b404f962386d5d51bdb744c6dada80c91089c73f --- /dev/null +++ b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/metrics.py @@ -0,0 +1,774 @@ +""" +This module contains ready-to-use functions that can be passed on to the +instrumentator instance with the `add()` method. The idea behind this is to +make the types of metrics you want to export with the instrumentation easily +customizable. The default instrumentation function `default` can also be found +here. + +If your requirements are really specific or very extensive it makes sense to +create your own instrumentation function instead of combining several functions +from this module. +""" + +from typing import Callable, List, Optional, Sequence, Tuple, Union + +from prometheus_client import REGISTRY, CollectorRegistry, Counter, Histogram, Summary +from starlette.requests import Request +from starlette.responses import Response + + +# ------------------------------------------------------------------------------ +class Info: + def __init__( + self, + request: Request, + response: Optional[Response], + method: str, + modified_handler: str, + modified_status: str, + modified_duration: float, + modified_duration_without_streaming: float = 0.0, + ): + """Creates Info object that is used for instrumentation functions. + + This is the only argument that is passed to the instrumentation functions. + + Args: + request (Request): Python Requests request object. + response (Response or None): Python Requests response object. + method (str): Unmodified method of the request. + modified_handler (str): Handler representation after processing by + instrumentator. For example grouped to `none` if not templated. + modified_status (str): Status code representation after processing + by instrumentator. For example grouping into `2xx`, `3xx` and so on. + modified_duration (float): Latency representation after processing + by instrumentator. For example rounding of decimals. Seconds. + modified_duration_without_streaming (float): Latency between request arrival and response starts (i.e. first chunk duration). + Excluding the streaming duration. Defaults to 0. + """ + + self.request = request + self.response = response + self.method = method + self.modified_handler = modified_handler + self.modified_status = modified_status + self.modified_duration = modified_duration + self.modified_duration_without_streaming = modified_duration_without_streaming + + +def _build_label_attribute_names( + should_include_handler: bool, + should_include_method: bool, + should_include_status: bool, +) -> Tuple[List[str], List[str]]: + """Builds up tuple with to be used label and attribute names. + + Args: + should_include_handler (bool): Should the `handler` label be part of the metric? + should_include_method (bool): Should the `method` label be part of the metric? + should_include_status (bool): Should the `status` label be part of the metric? + + Returns: + Tuple with two list elements. + + First element: List with all labels to be used. + Second element: List with all attribute names to be used from the + `Info` object. Done like this to enable dynamic on / off of labels. + """ + + label_names = [] + info_attribute_names = [] + + if should_include_handler: + label_names.append("handler") + info_attribute_names.append("modified_handler") + + if should_include_method: + label_names.append("method") + info_attribute_names.append("method") + + if should_include_status: + label_names.append("status") + info_attribute_names.append("modified_status") + + return label_names, info_attribute_names + + +def _is_duplicated_time_series(error: ValueError) -> bool: + return any( + map( + error.args[0].__contains__, + [ + "Duplicated timeseries in CollectorRegistry:", + "Duplicated time series in CollectorRegistry:", + ], + ) + ) + + +# ------------------------------------------------------------------------------ +# Instrumentation / Metrics functions + + +def latency( + metric_name: str = "http_request_duration_seconds", + metric_doc: str = "Duration of HTTP requests in seconds", + metric_namespace: str = "", + metric_subsystem: str = "", + should_include_handler: bool = True, + should_include_method: bool = True, + should_include_status: bool = True, + should_exclude_streaming_duration: bool = False, + buckets: Sequence[Union[float, str]] = Histogram.DEFAULT_BUCKETS, + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Default metric for the Prometheus Starlette Instrumentator. + + Args: + metric_name (str, optional): Name of the metric to be created. Must be + unique. Defaults to "http_request_duration_seconds". + + metric_doc (str, optional): Documentation of the metric. Defaults to + "Duration of HTTP requests in seconds". + + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + + should_include_handler: Should the `handler` label be part of the + metric? Defaults to `True`. + + should_include_method: Should the `method` label be part of the + metric? Defaults to `True`. + + should_include_status: Should the `status` label be part of the + metric? Defaults to `True`. + + should_exclude_streaming_duration: Should the streaming duration be + excluded? Defaults to `False`. + + buckets: Buckets for the histogram. Defaults to Prometheus default. + Defaults to default buckets from Prometheus client library. + + Returns: + Function that takes a single parameter `Info`. + """ + + if buckets[-1] != float("inf"): + buckets = [*buckets, float("inf")] + + label_names, info_attribute_names = _build_label_attribute_names( + should_include_handler, should_include_method, should_include_status + ) + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + if label_names: + METRIC = Histogram( + metric_name, + metric_doc, + labelnames=label_names, + buckets=buckets, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + else: + METRIC = Histogram( + metric_name, + metric_doc, + buckets=buckets, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + duration = info.modified_duration + if should_exclude_streaming_duration: + duration = info.modified_duration_without_streaming + else: + duration = info.modified_duration + + if label_names: + label_values = [ + getattr(info, attribute_name) + for attribute_name in info_attribute_names + ] + + METRIC.labels(*label_values).observe(duration) + else: + METRIC.observe(duration) + + return instrumentation + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None + + +def request_size( + metric_name: str = "http_request_size_bytes", + metric_doc: str = "Content bytes of requests.", + metric_namespace: str = "", + metric_subsystem: str = "", + should_include_handler: bool = True, + should_include_method: bool = True, + should_include_status: bool = True, + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Record the content length of incoming requests. + + If content length is missing 0 will be assumed. + + Args: + metric_name (str, optional): Name of the metric to be created. Must be + unique. Defaults to "http_request_size_bytes". + metric_doc (str, optional): Documentation of the metric. Defaults to + "Content bytes of requests.". + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + should_include_handler: Should the `handler` label be part of the + metric? Defaults to `True`. + should_include_method: Should the `method` label be part of the + metric? Defaults to `True`. + should_include_status: Should the `status` label be part of the metric? + Defaults to `True`. + + Returns: + Function that takes a single parameter `Info`. + """ + + label_names, info_attribute_names = _build_label_attribute_names( + should_include_handler, should_include_method, should_include_status + ) + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + if label_names: + METRIC = Summary( + metric_name, + metric_doc, + labelnames=label_names, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + else: + METRIC = Summary( + metric_name, + metric_doc, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + content_length = info.request.headers.get("Content-Length", 0) + if label_names: + label_values = [ + getattr(info, attribute_name) + for attribute_name in info_attribute_names + ] + + METRIC.labels(*label_values).observe(int(content_length)) + else: + METRIC.observe(int(content_length)) + + return instrumentation + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None + + +def response_size( + metric_name: str = "http_response_size_bytes", + metric_doc: str = "Content bytes of responses.", + metric_namespace: str = "", + metric_subsystem: str = "", + should_include_handler: bool = True, + should_include_method: bool = True, + should_include_status: bool = True, + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Record the content length of outgoing responses. + + If content length is missing 0 will be assumed. + + Args: + metric_name (str, optional): Name of the metric to be created. Must be + unique. Defaults to "http_response_size_bytes". + + metric_doc (str, optional): Documentation of the metric. Defaults to + "Content bytes of responses.". + + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + + should_include_handler: Should the `handler` label be part of the + metric? Defaults to `True`. + + should_include_method: Should the `method` label be part of the metric? + Defaults to `True`. + + should_include_status: Should the `status` label be part of the metric? + Defaults to `True`. + + Returns: + Function that takes a single parameter `Info`. + """ + + label_names, info_attribute_names = _build_label_attribute_names( + should_include_handler, should_include_method, should_include_status + ) + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + if label_names: + METRIC = Summary( + metric_name, + metric_doc, + labelnames=label_names, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + else: + METRIC = Summary( + metric_name, + metric_doc, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + if info.response and hasattr(info.response, "headers"): + content_length = info.response.headers.get("Content-Length", 0) + else: + content_length = 0 + + if label_names: + label_values = [ + getattr(info, attribute_name) + for attribute_name in info_attribute_names + ] + + METRIC.labels(*label_values).observe(int(content_length)) + else: + METRIC.observe(int(content_length)) + + return instrumentation + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None + + +def combined_size( + metric_name: str = "http_combined_size_bytes", + metric_doc: str = "Content bytes of requests and responses.", + metric_namespace: str = "", + metric_subsystem: str = "", + should_include_handler: bool = True, + should_include_method: bool = True, + should_include_status: bool = True, + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Record the combined content length of requests and responses. + + If content length is missing 0 will be assumed. + + Args: + metric_name (str, optional): Name of the metric to be created. Must be + unique. Defaults to "http_combined_size_bytes". + + metric_doc (str, optional): Documentation of the metric. Defaults to + "Content bytes of requests and responses.". + + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + + should_include_handler: Should the `handler` label be part of the + metric? Defaults to `True`. + + should_include_method: Should the `method` label be part of the metric? + Defaults to `True`. + + should_include_status: Should the `status` label be part of the metric? + Defaults to `True`. + + Returns: + Function that takes a single parameter `Info`. + """ + + label_names, info_attribute_names = _build_label_attribute_names( + should_include_handler, should_include_method, should_include_status + ) + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + if label_names: + METRIC = Summary( + metric_name, + metric_doc, + labelnames=label_names, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + else: + METRIC = Summary( + metric_name, + metric_doc, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + request_cl = info.request.headers.get("Content-Length", 0) + + if info.response and hasattr(info.response, "headers"): + response_cl = info.response.headers.get("Content-Length", 0) + else: + response_cl = 0 + + content_length = int(request_cl) + int(response_cl) + + if label_names: + label_values = [ + getattr(info, attribute_name) + for attribute_name in info_attribute_names + ] + + METRIC.labels(*label_values).observe(int(content_length)) + else: + METRIC.observe(int(content_length)) + + return instrumentation + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None + + +def requests( + metric_name: str = "http_requests_total", + metric_doc: str = "Total number of requests by method, status and handler.", + metric_namespace: str = "", + metric_subsystem: str = "", + should_include_handler: bool = True, + should_include_method: bool = True, + should_include_status: bool = True, + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Record the number of requests. + + Args: + metric_name (str, optional): Name of the metric to be created. Must + be unique. Defaults to "http_requests_total". + + metric_doc (str, optional): Documentation of the metric. Defaults to + "Total number of requests by method, status and handler.". + + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + + should_include_handler (bool, optional): Should the `handler` label + be part of the metric? Defaults to `True`. + + should_include_method (bool, optional): Should the `method` label be + part of the metric? Defaults to `True`. + + should_include_status (bool, optional): Should the `status` label be + part of the metric? Defaults to `True`. + + Returns: + Function that takes a single parameter `Info`. + """ + + label_names, info_attribute_names = _build_label_attribute_names( + should_include_handler, should_include_method, should_include_status + ) + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + if label_names: + METRIC = Counter( + metric_name, + metric_doc, + labelnames=label_names, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + else: + METRIC = Counter( + metric_name, + metric_doc, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + if label_names: + label_values = [ + getattr(info, attribute_name) + for attribute_name in info_attribute_names + ] + + METRIC.labels(*label_values).inc() + else: + METRIC.inc() + + return instrumentation + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None + + +def default( + metric_namespace: str = "", + metric_subsystem: str = "", + should_only_respect_2xx_for_highr: bool = False, + should_exclude_streaming_duration: bool = False, + latency_highr_buckets: Sequence[Union[float, str]] = ( + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.25, + 0.5, + 0.75, + 1, + 1.5, + 2, + 2.5, + 3, + 3.5, + 4, + 4.5, + 5, + 7.5, + 10, + 30, + 60, + ), + latency_lowr_buckets: Sequence[Union[float, str]] = (0.1, 0.5, 1), + registry: CollectorRegistry = REGISTRY, +) -> Optional[Callable[[Info], None]]: + """Contains multiple metrics to cover multiple things. + + Combines several metrics into a single function. Also more efficient than + multiple separate instrumentation functions that do more or less the same. + + You get the following: + + * `http_requests_total` (`handler`, `status`, `method`): Total number of + requests by handler, status and method. + * `http_request_size_bytes` (`handler`): Total number of incoming + content length bytes by handler. + * `http_response_size_bytes` (`handler`): Total number of outgoing + content length bytes by handler. + * `http_request_duration_highr_seconds` (no labels): High number of buckets + leading to more accurate calculation of percentiles. + * `http_request_duration_seconds` (`handler`, `method`): + Kepp the bucket count very low. Only put in SLIs. + + Args: + metric_namespace (str, optional): Namespace of all metrics in this + metric function. Defaults to "". + + metric_subsystem (str, optional): Subsystem of all metrics in this + metric function. Defaults to "". + + should_only_respect_2xx_for_highr (str, optional): Should the metric + `http_request_duration_highr_seconds` only include latencies of + requests / responses that have a status code starting with `2`? + Defaults to `False`. + + should_exclude_streaming_duration: Should the streaming duration be + excluded? Defaults to `False`. + + latency_highr_buckets (tuple[float], optional): Buckets tuple for high + res histogram. Can be large because no labels are used. Defaults to + (0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2, 2.5, + 3, 3.5, 4, 4.5, 5, 7.5, 10, 30, 60). + + latency_lowr_buckets (tuple[float], optional): Buckets tuple for low + res histogram. Should be very small as all possible labels are + included. Defaults to `(0.1, 0.5, 1)`. + + Returns: + Function that takes a single parameter `Info`. + """ + if latency_highr_buckets[-1] != float("inf"): + latency_highr_buckets = [*latency_highr_buckets, float("inf")] + + if latency_lowr_buckets[-1] != float("inf"): + latency_lowr_buckets = [*latency_lowr_buckets, float("inf")] + + # Starlette will call app.build_middleware_stack() with every new middleware + # added, which will call all this again, which will make the registry + # complain about duplicated metrics. + # + # The Python Prometheus client currently doesn't seem to have a way to + # verify if adding a metric will cause errors or not, so the only way to + # handle it seems to be with this try block. + try: + TOTAL = Counter( + name="http_requests_total", + documentation="Total number of requests by method, status and handler.", + labelnames=( + "method", + "status", + "handler", + ), + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + IN_SIZE = Summary( + name="http_request_size_bytes", + documentation=( + "Content length of incoming requests by handler. " + "Only value of header is respected. Otherwise ignored. " + "No percentile calculated. " + ), + labelnames=("handler",), + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + OUT_SIZE = Summary( + name="http_response_size_bytes", + documentation=( + "Content length of outgoing responses by handler. " + "Only value of header is respected. Otherwise ignored. " + "No percentile calculated. " + ), + labelnames=("handler",), + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + LATENCY_HIGHR = Histogram( + name="http_request_duration_highr_seconds", + documentation=( + "Latency with many buckets but no API specific labels. " + "Made for more accurate percentile calculations. " + ), + buckets=latency_highr_buckets, + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + LATENCY_LOWR = Histogram( + name="http_request_duration_seconds", + documentation=( + "Latency with only few buckets by handler. " + "Made to be only used if aggregation by handler is important. " + ), + buckets=latency_lowr_buckets, + labelnames=( + "method", + "handler", + ), + namespace=metric_namespace, + subsystem=metric_subsystem, + registry=registry, + ) + + def instrumentation(info: Info) -> None: + duration = info.modified_duration + if should_exclude_streaming_duration: + duration = info.modified_duration_without_streaming + else: + duration = info.modified_duration + + TOTAL.labels(info.method, info.modified_status, info.modified_handler).inc() + + IN_SIZE.labels(info.modified_handler).observe( + int(info.request.headers.get("Content-Length", 0)) + ) + + if info.response and hasattr(info.response, "headers"): + OUT_SIZE.labels(info.modified_handler).observe( + int(info.response.headers.get("Content-Length", 0)) + ) + else: + OUT_SIZE.labels(info.modified_handler).observe(0) + + if not should_only_respect_2xx_for_highr or info.modified_status.startswith( + "2" + ): + LATENCY_HIGHR.observe(duration) + + LATENCY_LOWR.labels( + handler=info.modified_handler, method=info.method + ).observe(duration) + + return instrumentation + + except ValueError as e: + if not _is_duplicated_time_series(e): + raise e + + return None diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/middleware.py b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/middleware.py new file mode 100644 index 0000000000000000000000000000000000000000..c1ae3d46d5abb22ca5c418fcf12ed8bd8292ddae --- /dev/null +++ b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/middleware.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import asyncio +import re +from http import HTTPStatus +from timeit import default_timer +from typing import Awaitable, Callable, Optional, Sequence, Tuple, Union + +from prometheus_client import REGISTRY, CollectorRegistry, Gauge +from starlette.applications import Starlette +from starlette.datastructures import Headers +from starlette.requests import Request +from starlette.responses import Response +from starlette.types import Message, Receive, Scope, Send + +from prometheus_fastapi_instrumentator import metrics, routing + + +class PrometheusInstrumentatorMiddleware: + def __init__( + self, + app: Starlette, + *, + should_group_status_codes: bool = True, + should_ignore_untemplated: bool = False, + should_group_untemplated: bool = True, + should_round_latency_decimals: bool = False, + should_respect_env_var: bool = False, + should_instrument_requests_inprogress: bool = False, + should_exclude_streaming_duration: bool = False, + excluded_handlers: Sequence[str] = (), + body_handlers: Sequence[str] = (), + round_latency_decimals: int = 4, + env_var_name: str = "ENABLE_METRICS", + inprogress_name: str = "http_requests_inprogress", + inprogress_labels: bool = False, + instrumentations: Sequence[Callable[[metrics.Info], None]] = (), + async_instrumentations: Sequence[Callable[[metrics.Info], Awaitable[None]]] = (), + metric_namespace: str = "", + metric_subsystem: str = "", + should_only_respect_2xx_for_highr: bool = False, + latency_highr_buckets: Sequence[Union[float, str]] = ( + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.25, + 0.5, + 0.75, + 1, + 1.5, + 2, + 2.5, + 3, + 3.5, + 4, + 4.5, + 5, + 7.5, + 10, + 30, + 60, + ), + latency_lowr_buckets: Sequence[Union[float, str]] = (0.1, 0.5, 1), + registry: CollectorRegistry = REGISTRY, + ) -> None: + self.app = app + + self.should_group_status_codes = should_group_status_codes + self.should_ignore_untemplated = should_ignore_untemplated + self.should_group_untemplated = should_group_untemplated + self.should_round_latency_decimals = should_round_latency_decimals + self.should_respect_env_var = should_respect_env_var + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + + self.round_latency_decimals = round_latency_decimals + self.env_var_name = env_var_name + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + self.registry = registry + + self.excluded_handlers = [re.compile(path) for path in excluded_handlers] + self.body_handlers = [re.compile(path) for path in body_handlers] + + if instrumentations: + self.instrumentations = instrumentations + else: + default_instrumentation = metrics.default( + metric_namespace=metric_namespace, + metric_subsystem=metric_subsystem, + should_only_respect_2xx_for_highr=should_only_respect_2xx_for_highr, + should_exclude_streaming_duration=should_exclude_streaming_duration, + latency_highr_buckets=latency_highr_buckets, + latency_lowr_buckets=latency_lowr_buckets, + registry=self.registry, + ) + if default_instrumentation: + self.instrumentations = [default_instrumentation] + else: + self.instrumentations = [] + + self.async_instrumentations = async_instrumentations + + self.inprogress: Optional[Gauge] = None + if self.should_instrument_requests_inprogress: + labels = ( + ( + "method", + "handler", + ) + if self.inprogress_labels + else () + ) + self.inprogress = Gauge( + name=self.inprogress_name, + documentation="Number of HTTP requests in progress.", + labelnames=labels, + multiprocess_mode="livesum", + ) + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + + request = Request(scope) + start_time = default_timer() + + handler, is_templated = self._get_handler(request) + is_excluded = self._is_handler_excluded(handler, is_templated) + handler = ( + "none" if not is_templated and self.should_group_untemplated else handler + ) + + if not is_excluded and self.inprogress: + if self.inprogress_labels: + inprogress = self.inprogress.labels(request.method, handler) + else: + inprogress = self.inprogress + inprogress.inc() + + status_code = 500 + headers = [] + body = b"" + response_start_time = None + + # Message body collected for handlers matching body_handlers patterns. + if any(pattern.search(handler) for pattern in self.body_handlers): + + async def send_wrapper(message: Message) -> None: + if message["type"] == "http.response.start": + nonlocal status_code, headers, response_start_time + headers = message["headers"] + status_code = message["status"] + response_start_time = default_timer() + elif message["type"] == "http.response.body" and message["body"]: + nonlocal body + body += message["body"] + await send(message) + + else: + + async def send_wrapper(message: Message) -> None: + if message["type"] == "http.response.start": + nonlocal status_code, headers, response_start_time + headers = message["headers"] + status_code = message["status"] + response_start_time = default_timer() + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception as exc: + raise exc + finally: + status = ( + str(status_code.value) + if isinstance(status_code, HTTPStatus) + else str(status_code) + ) + + if not is_excluded: + duration = max(default_timer() - start_time, 0.0) + duration_without_streaming = 0.0 + + if response_start_time: + duration_without_streaming = max( + response_start_time - start_time, 0.0 + ) + + if self.should_instrument_requests_inprogress: + inprogress.dec() + + if self.should_round_latency_decimals: + duration = round(duration, self.round_latency_decimals) + duration_without_streaming = round( + duration_without_streaming, self.round_latency_decimals + ) + + if self.should_group_status_codes: + status = status[0] + "xx" + + response = Response( + content=body, headers=Headers(raw=headers), status_code=status_code + ) + + info = metrics.Info( + request=request, + response=response, + method=request.method, + modified_handler=handler, + modified_status=status, + modified_duration=duration, + modified_duration_without_streaming=duration_without_streaming, + ) + + for instrumentation in self.instrumentations: + instrumentation(info) + + await asyncio.gather( + *[ + instrumentation(info) + for instrumentation in self.async_instrumentations + ] + ) + + def _get_handler(self, request: Request) -> Tuple[str, bool]: + """Extracts either template or (if no template) path. + + Args: + request (Request): Python Requests request object. + + Returns: + Tuple[str, bool]: Tuple with two elements. First element is either + template or if no template the path. Second element tells you + if the path is templated or not. + """ + route_name = routing.get_route_name(request) + return route_name or request.url.path, True if route_name else False + + def _is_handler_excluded(self, handler: str, is_templated: bool) -> bool: + """Determines if the handler should be ignored. + + Args: + handler (str): Handler that handles the request. + is_templated (bool): Shows if the request is templated. + + Returns: + bool: `True` if excluded, `False` if not. + """ + + if not is_templated and self.should_ignore_untemplated: + return True + + if any(pattern.search(handler) for pattern in self.excluded_handlers): + return True + + return False diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/py.typed b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/routing.py b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/routing.py new file mode 100644 index 0000000000000000000000000000000000000000..43d964f1bb14b3fd63e97a0445cad6b3aab85acc --- /dev/null +++ b/.venv/lib/python3.11/site-packages/prometheus_fastapi_instrumentator/routing.py @@ -0,0 +1,93 @@ +# BSD 3-Clause License +# +# Copyright (c) 2012, the Sentry Team, see AUTHORS for more details +# Copyright (c) 2019, Elasticsearch BV +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + +"""Helper module for routing. + +The two functions in this module are licensed under the BSD 3-Clause License +instead of the ISC License like the rest of the project. Therefore the code +is contained in a dedicated module. + +Based on code from [elastic/apm-agent-python](https://github.com/elastic/apm-agent-python/blob/527f62c0c50842f94ef90fda079853372539319a/elasticapm/contrib/starlette/__init__.py). +""" + +from typing import List, Optional + +from starlette.requests import HTTPConnection +from starlette.routing import Match, Mount, Route +from starlette.types import Scope + + +def _get_route_name( + scope: Scope, routes: List[Route], route_name: Optional[str] = None +) -> Optional[str]: + """Gets route name for given scope taking mounts into account.""" + + for route in routes: + match, child_scope = route.matches(scope) + if match == Match.FULL: + route_name = route.path + child_scope = {**scope, **child_scope} + if isinstance(route, Mount) and route.routes: + child_route_name = _get_route_name(child_scope, route.routes, route_name) + if child_route_name is None: + route_name = None + else: + route_name += child_route_name + return route_name + elif match == Match.PARTIAL and route_name is None: + route_name = route.path + return None + + +def get_route_name(request: HTTPConnection) -> Optional[str]: + """Gets route name for given request taking mounts into account.""" + + app = request.app + scope = request.scope + routes = app.routes + route_name = _get_route_name(scope, routes) + + # Starlette magically redirects requests if the path matches a route name + # with a trailing slash appended or removed. To not spam the transaction + # names list, we do the same here and put these redirects all in the + # same "redirect trailing slashes" transaction name. + if not route_name and app.router.redirect_slashes and scope["path"] != "/": + redirect_scope = dict(scope) + if scope["path"].endswith("/"): + redirect_scope["path"] = scope["path"][:-1] + trim = True + else: + redirect_scope["path"] = scope["path"] + "/" + trim = False + + route_name = _get_route_name(redirect_scope, routes) + if route_name is not None: + route_name = route_name + "/" if trim else route_name[:-1] + return route_name diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__init__.py b/.venv/lib/python3.11/site-packages/tiktoken/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3a531b1868633fff10adc94b47de27f25a9de052 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/__init__.py @@ -0,0 +1,6 @@ +# This is the public API of tiktoken +from .core import Encoding as Encoding +from .model import encoding_for_model as encoding_for_model +from .model import encoding_name_for_model as encoding_name_for_model +from .registry import get_encoding as get_encoding +from .registry import list_encoding_names as list_encoding_names diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/__init__.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8414b663a1a714ea2a2ba55d1276b8409f1e1031 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/__init__.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/_educational.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/_educational.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5ab0ffecda77ed6851be616d6850f64ed3d9ce72 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/_educational.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/core.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/core.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e011bbc20c4d412f020b0e402752da746b6cdf19 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/core.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/load.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/load.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2a51d8f8fe3bd8e43ba95467df8694f1cd09434 Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/load.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/model.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/model.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b1236b42f6ca67c25cc749aeb96ce2d995636be Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/model.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/registry.cpython-311.pyc b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/registry.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b5c9ab9c578ef656dac9e6051ea4327fc878c2d Binary files /dev/null and b/.venv/lib/python3.11/site-packages/tiktoken/__pycache__/registry.cpython-311.pyc differ diff --git a/.venv/lib/python3.11/site-packages/tiktoken/_educational.py b/.venv/lib/python3.11/site-packages/tiktoken/_educational.py new file mode 100644 index 0000000000000000000000000000000000000000..a6f5fccdd4c50e2b2a6291c6566c37b721b43924 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/_educational.py @@ -0,0 +1,221 @@ +"""This is an educational implementation of the byte pair encoding algorithm.""" +import collections +from typing import Optional + +import regex + +import tiktoken + + +class SimpleBytePairEncoding: + def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None: + """Creates an Encoding object.""" + # A regex pattern string that is used to split the input text + self.pat_str = pat_str + # A dictionary mapping token bytes to their ranks. The ranks correspond to merge priority + self.mergeable_ranks = mergeable_ranks + + self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()} + self._pat = regex.compile(pat_str) + + def encode(self, text: str, visualise: Optional[str] = "colour") -> list[int]: + """Encodes a string into tokens. + + >>> enc.encode("hello world") + [388, 372] + """ + # Use the regex to split the text into (approximately) words + words = self._pat.findall(text) + tokens = [] + for word in words: + # Turn each word into tokens, using the byte pair encoding algorithm + word_bytes = word.encode("utf-8") + word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, visualise=visualise) + tokens.extend(word_tokens) + return tokens + + def decode_bytes(self, tokens: list[int]) -> bytes: + """Decodes a list of tokens into bytes. + + >>> enc.decode_bytes([388, 372]) + b'hello world' + """ + return b"".join(self._decoder[token] for token in tokens) + + def decode(self, tokens: list[int]) -> str: + """Decodes a list of tokens into a string. + + Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace + the invalid bytes with the replacement character "�". + + >>> enc.decode([388, 372]) + 'hello world' + """ + return self.decode_bytes(tokens).decode("utf-8", errors="replace") + + def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]: + """Decodes a list of tokens into a list of bytes. + + Useful for visualising how a string is tokenised. + + >>> enc.decode_tokens_bytes([388, 372]) + [b'hello', b' world'] + """ + return [self._decoder[token] for token in tokens] + + @staticmethod + def train(training_data: str, vocab_size: int, pat_str: str): + """Train a BPE tokeniser on some data!""" + mergeable_ranks = bpe_train(data=training_data, vocab_size=vocab_size, pat_str=pat_str) + return SimpleBytePairEncoding(pat_str=pat_str, mergeable_ranks=mergeable_ranks) + + @staticmethod + def from_tiktoken(encoding): + if isinstance(encoding, str): + encoding = tiktoken.get_encoding(encoding) + return SimpleBytePairEncoding( + pat_str=encoding._pat_str, mergeable_ranks=encoding._mergeable_ranks + ) + + +def bpe_encode( + mergeable_ranks: dict[bytes, int], input: bytes, visualise: Optional[str] = "colour" +) -> list[int]: + parts = [bytes([b]) for b in input] + while True: + # See the intermediate merges play out! + if visualise: + if visualise in ["colour", "color"]: + visualise_tokens(parts) + elif visualise == "simple": + print(parts) + + # Iterate over all pairs and find the pair we want to merge the most + min_idx = None + min_rank = None + for i, pair in enumerate(zip(parts[:-1], parts[1:])): + rank = mergeable_ranks.get(pair[0] + pair[1]) + if rank is not None and (min_rank is None or rank < min_rank): + min_idx = i + min_rank = rank + + # If there were no pairs we could merge, we're done! + if min_rank is None: + break + assert min_idx is not None + + # Otherwise, merge that pair and leave the rest unchanged. Then repeat. + parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :] + + if visualise: + print() + + tokens = [mergeable_ranks[part] for part in parts] + return tokens + + +def bpe_train( + data: str, vocab_size: int, pat_str: str, visualise: Optional[str] = "colour" +) -> dict[bytes, int]: + # First, add tokens for each individual byte value + if vocab_size < 2**8: + raise ValueError("vocab_size must be at least 256, so we can encode all bytes") + ranks = {} + for i in range(2**8): + ranks[bytes([i])] = i + + # Splinter up our data into lists of bytes + # data = "Hello world" + # words = [ + # [b'H', b'e', b'l', b'l', b'o'], + # [b' ', b'w', b'o', b'r', b'l', b'd'] + # ] + words: list[list[bytes]] = [ + [bytes([b]) for b in word.encode("utf-8")] for word in regex.findall(pat_str, data) + ] + + # Now, use our data to figure out which merges we should make + while len(ranks) < vocab_size: + # Find the most common pair. This will become our next token + stats = collections.Counter() + for piece in words: + for pair in zip(piece[:-1], piece[1:]): + stats[pair] += 1 + + most_common_pair = max(stats, key=lambda x: stats[x]) + token_bytes = most_common_pair[0] + most_common_pair[1] + token = len(ranks) + # Add the new token! + ranks[token_bytes] = token + + # Now merge that most common pair in all the words. That is, update our training data + # to reflect our decision to make that pair into a new token. + new_words = [] + for word in words: + new_word = [] + i = 0 + while i < len(word) - 1: + if (word[i], word[i + 1]) == most_common_pair: + # We found our pair! Merge it + new_word.append(token_bytes) + i += 2 + else: + new_word.append(word[i]) + i += 1 + if i == len(word) - 1: + new_word.append(word[i]) + new_words.append(new_word) + words = new_words + + # See the intermediate merges play out! + if visualise: + print(f"The current most common pair is {most_common_pair[0]} + {most_common_pair[1]}") + print(f"So we made {token_bytes} our {len(ranks)}th token") + if visualise in ["colour", "color"]: + print("Now the first fifty words in our training data look like:") + visualise_tokens([token for word in words[:50] for token in word]) + elif visualise == "simple": + print("Now the first twenty words in our training data look like:") + for word in words[:20]: + print(word) + print("\n") + + return ranks + + +def visualise_tokens(token_values: list[bytes]) -> None: + background = [f"\u001b[48;5;{i}m" for i in [167, 179, 185, 77, 80, 68, 134]] + # If token boundaries do not occur at unicode character boundaries, it's unclear how best to + # visualise the token. Here, we'll just use the unicode replacement character to represent some + # fraction of a character. + unicode_token_values = [x.decode("utf-8", errors="replace") for x in token_values] + + running_length = 0 + last_color = None + for token in unicode_token_values: + color = background[running_length % len(background)] + if color == last_color: + color = background[(running_length + 1) % len(background)] + assert color != last_color + last_color = color + running_length += len(token) + print(color + token, end="") + print("\u001b[0m") + + +def train_simple_encoding(): + gpt2_pattern = ( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + with open(__file__, "r") as f: + data = f.read() + + enc = SimpleBytePairEncoding.train(data, vocab_size=600, pat_str=gpt2_pattern) + + print("This is the sequence of merges performed in order to encode 'hello world':") + tokens = enc.encode("hello world") + assert enc.decode(tokens) == "hello world" + assert enc.decode_bytes(tokens) == b"hello world" + assert enc.decode_tokens_bytes(tokens) == [b"hello", b" world"] + + return enc diff --git a/.venv/lib/python3.11/site-packages/tiktoken/core.py b/.venv/lib/python3.11/site-packages/tiktoken/core.py new file mode 100644 index 0000000000000000000000000000000000000000..3e32d4ce6addb8e88256a2b913b1266072f614c4 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/core.py @@ -0,0 +1,407 @@ +from __future__ import annotations + +import functools +from concurrent.futures import ThreadPoolExecutor +from typing import AbstractSet, Collection, Literal, NoReturn, Optional, Union + +import regex + +from tiktoken import _tiktoken + + +class Encoding: + def __init__( + self, + name: str, + *, + pat_str: str, + mergeable_ranks: dict[bytes, int], + special_tokens: dict[str, int], + explicit_n_vocab: Optional[int] = None, + ): + """Creates an Encoding object. + + See openai_public.py for examples of how to construct an Encoding object. + + Args: + name: The name of the encoding. It should be clear from the name of the encoding + what behaviour to expect, in particular, encodings with different special tokens + should have different names. + pat_str: A regex pattern string that is used to split the input text. + mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks + must correspond to merge priority. + special_tokens: A dictionary mapping special token strings to their token values. + explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked + that the number of mergeable tokens and special tokens is equal to this number. + """ + self.name = name + + self._pat_str = pat_str + self._mergeable_ranks = mergeable_ranks + self._special_tokens = special_tokens + + self.max_token_value = max( + max(mergeable_ranks.values()), max(special_tokens.values(), default=0) + ) + if explicit_n_vocab: + assert len(mergeable_ranks) + len(special_tokens) == explicit_n_vocab + assert self.max_token_value == explicit_n_vocab - 1 + + self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str) + + def __repr__(self) -> str: + return f"" + + # ==================== + # Encoding + # ==================== + + def encode_ordinary(self, text: str) -> list[int]: + """Encodes a string into tokens, ignoring special tokens. + + This is equivalent to `encode(text, disallowed_special=())` (but slightly faster). + + ``` + >>> enc.encode_ordinary("hello world") + [31373, 995] + """ + try: + return self._core_bpe.encode_ordinary(text) + except UnicodeEncodeError: + # See comment in encode + text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return self._core_bpe.encode_ordinary(text) + + def encode( + self, + text: str, + *, + allowed_special: Union[Literal["all"], AbstractSet[str]] = set(), # noqa: B006 + disallowed_special: Union[Literal["all"], Collection[str]] = "all", + ) -> list[int]: + """Encodes a string into tokens. + + Special tokens are artificial tokens used to unlock capabilities from a model, + such as fill-in-the-middle. So we want to be careful about accidentally encoding special + tokens, since they can be used to trick a model into doing something we don't want it to do. + + Hence, by default, encode will raise an error if it encounters text that corresponds + to a special token. This can be controlled on a per-token level using the `allowed_special` + and `disallowed_special` parameters. In particular: + - Setting `disallowed_special` to () will prevent this function from raising errors and + cause all text corresponding to special tokens to be encoded as natural text. + - Setting `allowed_special` to "all" will cause this function to treat all text + corresponding to special tokens to be encoded as special tokens. + + ``` + >>> enc.encode("hello world") + [31373, 995] + >>> enc.encode("<|endoftext|>", allowed_special={"<|endoftext|>"}) + [50256] + >>> enc.encode("<|endoftext|>", allowed_special="all") + [50256] + >>> enc.encode("<|endoftext|>") + # Raises ValueError + >>> enc.encode("<|endoftext|>", disallowed_special=()) + [27, 91, 437, 1659, 5239, 91, 29] + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if disallowed_special: + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + if match := _special_token_regex(disallowed_special).search(text): + raise_disallowed_special_token(match.group()) + + # https://github.com/PyO3/pyo3/pull/3632 + if isinstance(allowed_special, frozenset): + allowed_special = set(allowed_special) + + try: + return self._core_bpe.encode(text, allowed_special) + except UnicodeEncodeError: + # BPE operates on bytes, but the regex operates on unicode. If we pass a str that is + # invalid UTF-8 to Rust, it will rightfully complain. Here we do a quick and dirty + # fixup for any surrogate pairs that may have sneaked their way into the text. + # Technically, this introduces a place where encode + decode doesn't roundtrip a Python + # string, but given that this is input we want to support, maybe that's okay. + # Also we use errors="replace" to handle weird things like lone surrogates. + text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return self._core_bpe.encode(text, allowed_special) + + def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> list[list[int]]: + """Encodes a list of strings into tokens, in parallel, ignoring special tokens. + + This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster). + + ``` + >>> enc.encode_ordinary_batch(["hello world", "goodbye world"]) + [[31373, 995], [11274, 16390, 995]] + ``` + """ + encoder = functools.partial(self.encode_ordinary) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(encoder, text)) + + def encode_batch( + self, + text: list[str], + *, + num_threads: int = 8, + allowed_special: Union[Literal["all"], AbstractSet[str]] = set(), # noqa: B006 + disallowed_special: Union[Literal["all"], Collection[str]] = "all", + ) -> list[list[int]]: + """Encodes a list of strings into tokens, in parallel. + + See `encode` for more details on `allowed_special` and `disallowed_special`. + + ``` + >>> enc.encode_batch(["hello world", "goodbye world"]) + [[31373, 995], [11274, 16390, 995]] + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + + encoder = functools.partial( + self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special + ) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(encoder, text)) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: Union[Literal["all"], AbstractSet[str]] = set(), # noqa: B006 + disallowed_special: Union[Literal["all"], Collection[str]] = "all", + ) -> tuple[list[int], list[list[int]]]: + """Encodes a string into stable tokens and possible completion sequences. + + Note that the stable tokens will only represent a substring of `text`. + + See `encode` for more details on `allowed_special` and `disallowed_special`. + + This API should itself be considered unstable. + + ``` + >>> enc.encode_with_unstable("hello fanta") + ([31373], [(277, 4910), (5113, 265), ..., (8842,)]) + + >>> text = "..." + >>> stable_tokens, completions = enc.encode_with_unstable(text) + >>> assert text.encode().startswith(enc.decode_bytes(stable_tokens)) + >>> assert all(enc.decode_bytes(stable_tokens + seq).startswith(text.encode()) for seq in completions) + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if disallowed_special: + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + if match := _special_token_regex(disallowed_special).search(text): + raise_disallowed_special_token(match.group()) + + return self._core_bpe.encode_with_unstable(text, allowed_special) + + def encode_single_token(self, text_or_bytes: Union[str, bytes]) -> int: + """Encodes text corresponding to a single token to its token value. + + NOTE: this will encode all special tokens. + + Raises `KeyError` if the token is not in the vocabulary. + + ``` + >>> enc.encode_single_token("hello") + 31373 + ``` + """ + if isinstance(text_or_bytes, str): + text_or_bytes = text_or_bytes.encode("utf-8") + return self._core_bpe.encode_single_token(text_or_bytes) + + # ==================== + # Decoding + # ==================== + + def decode_bytes(self, tokens: list[int]) -> bytes: + """Decodes a list of tokens into bytes. + + ``` + >>> enc.decode_bytes([31373, 995]) + b'hello world' + ``` + """ + return self._core_bpe.decode_bytes(tokens) + + def decode(self, tokens: list[int], errors: str = "replace") -> str: + """Decodes a list of tokens into a string. + + WARNING: the default behaviour of this function is lossy, since decoded bytes are not + guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter, + for instance, setting `errors=strict`. + + ``` + >>> enc.decode([31373, 995]) + 'hello world' + ``` + """ + return self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + """Decodes a token into bytes. + + NOTE: this will decode all special tokens. + + Raises `KeyError` if the token is not in the vocabulary. + + ``` + >>> enc.decode_single_token_bytes(31373) + b'hello' + ``` + """ + return self._core_bpe.decode_single_token_bytes(token) + + def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]: + """Decodes a list of tokens into a list of bytes. + + Useful for visualising tokenisation. + >>> enc.decode_tokens_bytes([31373, 995]) + [b'hello', b' world'] + """ + return [self.decode_single_token_bytes(token) for token in tokens] + + def decode_with_offsets(self, tokens: list[int]) -> tuple[str, list[int]]: + """Decodes a list of tokens into a string and a list of offsets. + + Each offset is the index into text corresponding to the start of each token. + If UTF-8 character boundaries do not line up with token boundaries, the offset is the index + of the first character that contains bytes from the token. + + This will currently raise if given tokens that decode to invalid UTF-8; this behaviour may + change in the future to be more permissive. + + >>> enc.decode_with_offsets([31373, 995]) + ('hello world', [0, 5]) + """ + token_bytes = self.decode_tokens_bytes(tokens) + + text_len = 0 + offsets = [] + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + + # TODO: assess correctness for errors="ignore" and errors="replace" + text = b"".join(token_bytes).decode("utf-8", errors="strict") + return text, offsets + + def decode_batch( + self, batch: list[list[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: + """Decodes a batch (list of lists of tokens) into a list of strings.""" + decoder = functools.partial(self.decode, errors=errors) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(decoder, batch)) + + def decode_bytes_batch(self, batch: list[list[int]], *, num_threads: int = 8) -> list[bytes]: + """Decodes a batch (list of lists of tokens) into a list of bytes.""" + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(self.decode_bytes, batch)) + + # ==================== + # Miscellaneous + # ==================== + + def token_byte_values(self) -> list[bytes]: + """Returns the list of all token byte values.""" + return self._core_bpe.token_byte_values() + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @functools.cached_property + def special_tokens_set(self) -> set[str]: + return set(self._special_tokens.keys()) + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + # ==================== + # Private + # ==================== + + def _encode_single_piece(self, text_or_bytes: Union[str, bytes]) -> list[int]: + """Encodes text corresponding to bytes without a regex split. + + NOTE: this will not encode any special tokens. + + ``` + >>> enc.encode_single_piece("helloqqqq") + [31373, 38227, 38227] + ``` + """ + if isinstance(text_or_bytes, str): + text_or_bytes = text_or_bytes.encode("utf-8") + return self._core_bpe.encode_single_piece(text_or_bytes) + + def _encode_only_native_bpe(self, text: str) -> list[int]: + """Encodes a string into tokens, but do regex splitting in Python.""" + _unused_pat = regex.compile(self._pat_str) + ret = [] + for piece in regex.findall(_unused_pat, text): + ret.extend(self._core_bpe.encode_single_piece(piece)) + return ret + + def _encode_bytes(self, text: bytes) -> list[int]: + return self._core_bpe._encode_bytes(text) + + def __getstate__(self) -> object: + import tiktoken.registry + + # As an optimisation, pickle registered encodings by reference + if self is tiktoken.registry.ENCODINGS.get(self.name): + return self.name + return { + "name": self.name, + "pat_str": self._pat_str, + "mergeable_ranks": self._mergeable_ranks, + "special_tokens": self._special_tokens, + } + + def __setstate__(self, value: object) -> None: + import tiktoken.registry + + if isinstance(value, str): + self.__dict__ = tiktoken.registry.get_encoding(value).__dict__ + return + self.__init__(**value) + + +@functools.lru_cache(maxsize=128) +def _special_token_regex(tokens: frozenset[str]) -> "regex.Pattern[str]": + inner = "|".join(regex.escape(token) for token in tokens) + return regex.compile(f"({inner})") + + +def raise_disallowed_special_token(token: str) -> NoReturn: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + f"If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) diff --git a/.venv/lib/python3.11/site-packages/tiktoken/load.py b/.venv/lib/python3.11/site-packages/tiktoken/load.py new file mode 100644 index 0000000000000000000000000000000000000000..cc0a6a6de608471b06b3b200e410ad309c0bfa9c --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/load.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import tempfile +import uuid +from typing import Optional + +import requests + + +def read_file(blobpath: str) -> bytes: + if not blobpath.startswith("http://") and not blobpath.startswith("https://"): + try: + import blobfile + except ImportError as e: + raise ImportError( + "blobfile is not installed. Please install it by running `pip install blobfile`." + ) from e + with blobfile.BlobFile(blobpath, "rb") as f: + return f.read() + # avoiding blobfile for public files helps avoid auth issues, like MFA prompts + resp = requests.get(blobpath) + resp.raise_for_status() + return resp.content + + +def check_hash(data: bytes, expected_hash: str) -> bool: + actual_hash = hashlib.sha256(data).hexdigest() + return actual_hash == expected_hash + + +def read_file_cached(blobpath: str, expected_hash: Optional[str] = None) -> bytes: + user_specified_cache = True + if "TIKTOKEN_CACHE_DIR" in os.environ: + cache_dir = os.environ["TIKTOKEN_CACHE_DIR"] + elif "DATA_GYM_CACHE_DIR" in os.environ: + cache_dir = os.environ["DATA_GYM_CACHE_DIR"] + else: + cache_dir = os.path.join(tempfile.gettempdir(), "data-gym-cache") + user_specified_cache = False + + if cache_dir == "": + # disable caching + return read_file(blobpath) + + cache_key = hashlib.sha1(blobpath.encode()).hexdigest() + + cache_path = os.path.join(cache_dir, cache_key) + if os.path.exists(cache_path): + with open(cache_path, "rb") as f: + data = f.read() + if expected_hash is None or check_hash(data, expected_hash): + return data + + # the cached file does not match the hash, remove it and re-fetch + try: + os.remove(cache_path) + except OSError: + pass + + contents = read_file(blobpath) + if expected_hash and not check_hash(contents, expected_hash): + raise ValueError( + f"Hash mismatch for data downloaded from {blobpath} (expected {expected_hash}). " + f"This may indicate a corrupted download. Please try again." + ) + + try: + os.makedirs(cache_dir, exist_ok=True) + tmp_filename = cache_path + "." + str(uuid.uuid4()) + ".tmp" + with open(tmp_filename, "wb") as f: + f.write(contents) + os.rename(tmp_filename, cache_path) + except OSError: + # don't raise if we can't write to the default cache, e.g. issue #75 + if user_specified_cache: + raise + + return contents + + +def data_gym_to_mergeable_bpe_ranks( + vocab_bpe_file: str, + encoder_json_file: str, + vocab_bpe_hash: Optional[str] = None, + encoder_json_hash: Optional[str] = None, +) -> dict[bytes, int]: + # NB: do not add caching to this function + rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "] + + data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte} + n = 0 + for b in range(2**8): + if b not in rank_to_intbyte: + rank_to_intbyte.append(b) + data_gym_byte_to_byte[chr(2**8 + n)] = b + n += 1 + assert len(rank_to_intbyte) == 2**8 + + # vocab_bpe contains the merges along with associated ranks + vocab_bpe_contents = read_file_cached(vocab_bpe_file, vocab_bpe_hash).decode() + bpe_merges = [tuple(merge_str.split()) for merge_str in vocab_bpe_contents.split("\n")[1:-1]] + + def decode_data_gym(value: str) -> bytes: + return bytes(data_gym_byte_to_byte[b] for b in value) + + # add the single byte tokens + bpe_ranks = {bytes([b]): i for i, b in enumerate(rank_to_intbyte)} + # add the merged tokens + n = len(bpe_ranks) + for first, second in bpe_merges: + bpe_ranks[decode_data_gym(first) + decode_data_gym(second)] = n + n += 1 + + # check that the encoder file matches the merges file + # this sanity check is important since tiktoken assumes that ranks are ordered the same + # as merge priority + encoder_json = json.loads(read_file_cached(encoder_json_file, encoder_json_hash)) + encoder_json_loaded = {decode_data_gym(k): v for k, v in encoder_json.items()} + # drop these two special tokens if present, since they're not mergeable bpe tokens + encoder_json_loaded.pop(b"<|endoftext|>", None) + encoder_json_loaded.pop(b"<|startoftext|>", None) + assert bpe_ranks == encoder_json_loaded + + return bpe_ranks + + +def dump_tiktoken_bpe(bpe_ranks: dict[bytes, int], tiktoken_bpe_file: str) -> None: + try: + import blobfile + except ImportError as e: + raise ImportError( + "blobfile is not installed. Please install it by running `pip install blobfile`." + ) from e + with blobfile.BlobFile(tiktoken_bpe_file, "wb") as f: + for token, rank in sorted(bpe_ranks.items(), key=lambda x: x[1]): + f.write(base64.b64encode(token) + b" " + str(rank).encode() + b"\n") + + +def load_tiktoken_bpe( + tiktoken_bpe_file: str, expected_hash: Optional[str] = None +) -> dict[bytes, int]: + # NB: do not add caching to this function + contents = read_file_cached(tiktoken_bpe_file, expected_hash) + return { + base64.b64decode(token): int(rank) + for token, rank in (line.split() for line in contents.splitlines() if line) + } diff --git a/.venv/lib/python3.11/site-packages/tiktoken/model.py b/.venv/lib/python3.11/site-packages/tiktoken/model.py new file mode 100644 index 0000000000000000000000000000000000000000..6ecd7232219a02148f7a88e39a5a04389ad8abde --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/model.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from .core import Encoding +from .registry import get_encoding + +# TODO: these will likely be replaced by an API endpoint +MODEL_PREFIX_TO_ENCODING: dict[str, str] = { + # chat + "gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13 + "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k + "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc. + "gpt-35-turbo-": "cl100k_base", # Azure deployment name + # fine-tuned + "ft:gpt-4": "cl100k_base", + "ft:gpt-3.5-turbo": "cl100k_base", + "ft:davinci-002": "cl100k_base", + "ft:babbage-002": "cl100k_base", +} + +MODEL_TO_ENCODING: dict[str, str] = { + # chat + "gpt-4o": "o200k_base", + "gpt-4": "cl100k_base", + "gpt-3.5-turbo": "cl100k_base", + "gpt-3.5": "cl100k_base", # Common shorthand + "gpt-35-turbo": "cl100k_base", # Azure deployment name + # base + "davinci-002": "cl100k_base", + "babbage-002": "cl100k_base", + # embeddings + "text-embedding-ada-002": "cl100k_base", + "text-embedding-3-small": "cl100k_base", + "text-embedding-3-large": "cl100k_base", + # DEPRECATED MODELS + # text (DEPRECATED) + "text-davinci-003": "p50k_base", + "text-davinci-002": "p50k_base", + "text-davinci-001": "r50k_base", + "text-curie-001": "r50k_base", + "text-babbage-001": "r50k_base", + "text-ada-001": "r50k_base", + "davinci": "r50k_base", + "curie": "r50k_base", + "babbage": "r50k_base", + "ada": "r50k_base", + # code (DEPRECATED) + "code-davinci-002": "p50k_base", + "code-davinci-001": "p50k_base", + "code-cushman-002": "p50k_base", + "code-cushman-001": "p50k_base", + "davinci-codex": "p50k_base", + "cushman-codex": "p50k_base", + # edit (DEPRECATED) + "text-davinci-edit-001": "p50k_edit", + "code-davinci-edit-001": "p50k_edit", + # old embeddings (DEPRECATED) + "text-similarity-davinci-001": "r50k_base", + "text-similarity-curie-001": "r50k_base", + "text-similarity-babbage-001": "r50k_base", + "text-similarity-ada-001": "r50k_base", + "text-search-davinci-doc-001": "r50k_base", + "text-search-curie-doc-001": "r50k_base", + "text-search-babbage-doc-001": "r50k_base", + "text-search-ada-doc-001": "r50k_base", + "code-search-babbage-code-001": "r50k_base", + "code-search-ada-code-001": "r50k_base", + # open source + "gpt2": "gpt2", + "gpt-2": "gpt2", # Maintains consistency with gpt-4 +} + + +def encoding_name_for_model(model_name: str) -> str: + """Returns the name of the encoding used by a model. + + Raises a KeyError if the model name is not recognised. + """ + encoding_name = None + if model_name in MODEL_TO_ENCODING: + encoding_name = MODEL_TO_ENCODING[model_name] + else: + # Check if the model matches a known prefix + # Prefix matching avoids needing library updates for every model version release + # Note that this can match on non-existent models (e.g., gpt-3.5-turbo-FAKE) + for model_prefix, model_encoding_name in MODEL_PREFIX_TO_ENCODING.items(): + if model_name.startswith(model_prefix): + return model_encoding_name + + if encoding_name is None: + raise KeyError( + f"Could not automatically map {model_name} to a tokeniser. " + "Please use `tiktoken.get_encoding` to explicitly get the tokeniser you expect." + ) from None + + return encoding_name + + +def encoding_for_model(model_name: str) -> Encoding: + """Returns the encoding used by a model. + + Raises a KeyError if the model name is not recognised. + """ + return get_encoding(encoding_name_for_model(model_name)) diff --git a/.venv/lib/python3.11/site-packages/tiktoken/py.typed b/.venv/lib/python3.11/site-packages/tiktoken/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/.venv/lib/python3.11/site-packages/tiktoken/registry.py b/.venv/lib/python3.11/site-packages/tiktoken/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..a753ce6776b6893267941d0e81e358ff9959712b --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tiktoken/registry.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import functools +import importlib +import pkgutil +import threading +from typing import Any, Callable, Optional, Sequence + +import tiktoken_ext + +from tiktoken.core import Encoding + +_lock = threading.RLock() +ENCODINGS: dict[str, Encoding] = {} +ENCODING_CONSTRUCTORS: Optional[dict[str, Callable[[], dict[str, Any]]]] = None + + +@functools.lru_cache() +def _available_plugin_modules() -> Sequence[str]: + # tiktoken_ext is a namespace package + # submodules inside tiktoken_ext will be inspected for ENCODING_CONSTRUCTORS attributes + # - we use namespace package pattern so `pkgutil.iter_modules` is fast + # - it's a separate top-level package because namespace subpackages of non-namespace + # packages don't quite do what you want with editable installs + mods = [] + plugin_mods = pkgutil.iter_modules(tiktoken_ext.__path__, tiktoken_ext.__name__ + ".") + for _, mod_name, _ in plugin_mods: + mods.append(mod_name) + return mods + + +def _find_constructors() -> None: + global ENCODING_CONSTRUCTORS + with _lock: + if ENCODING_CONSTRUCTORS is not None: + return + ENCODING_CONSTRUCTORS = {} + + for mod_name in _available_plugin_modules(): + mod = importlib.import_module(mod_name) + try: + constructors = mod.ENCODING_CONSTRUCTORS + except AttributeError as e: + raise ValueError( + f"tiktoken plugin {mod_name} does not define ENCODING_CONSTRUCTORS" + ) from e + for enc_name, constructor in constructors.items(): + if enc_name in ENCODING_CONSTRUCTORS: + raise ValueError( + f"Duplicate encoding name {enc_name} in tiktoken plugin {mod_name}" + ) + ENCODING_CONSTRUCTORS[enc_name] = constructor + + +def get_encoding(encoding_name: str) -> Encoding: + if encoding_name in ENCODINGS: + return ENCODINGS[encoding_name] + + with _lock: + if encoding_name in ENCODINGS: + return ENCODINGS[encoding_name] + + if ENCODING_CONSTRUCTORS is None: + _find_constructors() + assert ENCODING_CONSTRUCTORS is not None + + if encoding_name not in ENCODING_CONSTRUCTORS: + raise ValueError( + f"Unknown encoding {encoding_name}. Plugins found: {_available_plugin_modules()}" + ) + + constructor = ENCODING_CONSTRUCTORS[encoding_name] + enc = Encoding(**constructor()) + ENCODINGS[encoding_name] = enc + return enc + + +def list_encoding_names() -> list[str]: + with _lock: + if ENCODING_CONSTRUCTORS is None: + _find_constructors() + assert ENCODING_CONSTRUCTORS is not None + return list(ENCODING_CONSTRUCTORS) diff --git a/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/INSTALLER b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/METADATA b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..2d69810e1bc2b9bb812387693abcf7b5a15fef5e --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/METADATA @@ -0,0 +1,209 @@ +Metadata-Version: 2.3 +Name: tokenizers +Version: 0.21.0 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Requires-Dist: huggingface-hub >=0.16.4, <1.0 +Requires-Dist: pytest ; extra == 'testing' +Requires-Dist: requests ; extra == 'testing' +Requires-Dist: numpy ; extra == 'testing' +Requires-Dist: datasets ; extra == 'testing' +Requires-Dist: black ==22.3 ; extra == 'testing' +Requires-Dist: ruff ; extra == 'testing' +Requires-Dist: sphinx ; extra == 'docs' +Requires-Dist: sphinx-rtd-theme ; extra == 'docs' +Requires-Dist: setuptools-rust ; extra == 'docs' +Requires-Dist: tokenizers[testing] ; extra == 'dev' +Provides-Extra: testing +Provides-Extra: docs +Provides-Extra: dev +Keywords: NLP,tokenizer,BPE,transformer,deep learning +Author: Anthony MOI +Author-email: Nicolas Patry , Anthony Moi +Requires-Python: >=3.7 +Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM +Project-URL: Homepage, https://github.com/huggingface/tokenizers +Project-URL: Source, https://github.com/huggingface/tokenizers + +

+
+ +
+

+

+ + Build + + + GitHub + +

+
+ +# Tokenizers + +Provides an implementation of today's most used tokenizers, with a focus on performance and +versatility. + +Bindings over the [Rust](https://github.com/huggingface/tokenizers/tree/master/tokenizers) implementation. +If you are interested in the High-level design, you can go check it there. + +Otherwise, let's dive in! + +## Main features: + + - Train new vocabularies and tokenize using 4 pre-made tokenizers (Bert WordPiece and the 3 + most common BPE versions). + - Extremely fast (both training and tokenization), thanks to the Rust implementation. Takes + less than 20 seconds to tokenize a GB of text on a server's CPU. + - Easy to use, but also extremely versatile. + - Designed for research and production. + - Normalization comes with alignments tracking. It's always possible to get the part of the + original sentence that corresponds to a given token. + - Does all the pre-processing: Truncate, Pad, add the special tokens your model needs. + +### Installation + +#### With pip: + +```bash +pip install tokenizers +``` + +#### From sources: + +To use this method, you need to have the Rust installed: + +```bash +# Install with: +curl https://sh.rustup.rs -sSf | sh -s -- -y +export PATH="$HOME/.cargo/bin:$PATH" +``` + +Once Rust is installed, you can compile doing the following + +```bash +git clone https://github.com/huggingface/tokenizers +cd tokenizers/bindings/python + +# Create a virtual env (you can use yours as well) +python -m venv .env +source .env/bin/activate + +# Install `tokenizers` in the current virtual env +pip install -e . +``` + +### Load a pretrained tokenizer from the Hub + +```python +from tokenizers import Tokenizer + +tokenizer = Tokenizer.from_pretrained("bert-base-cased") +``` + +### Using the provided Tokenizers + +We provide some pre-build tokenizers to cover the most common cases. You can easily load one of +these using some `vocab.json` and `merges.txt` files: + +```python +from tokenizers import CharBPETokenizer + +# Initialize a tokenizer +vocab = "./path/to/vocab.json" +merges = "./path/to/merges.txt" +tokenizer = CharBPETokenizer(vocab, merges) + +# And then encode: +encoded = tokenizer.encode("I can feel the magic, can you?") +print(encoded.ids) +print(encoded.tokens) +``` + +And you can train them just as simply: + +```python +from tokenizers import CharBPETokenizer + +# Initialize a tokenizer +tokenizer = CharBPETokenizer() + +# Then train it! +tokenizer.train([ "./path/to/files/1.txt", "./path/to/files/2.txt" ]) + +# Now, let's use it: +encoded = tokenizer.encode("I can feel the magic, can you?") + +# And finally save it somewhere +tokenizer.save("./path/to/directory/my-bpe.tokenizer.json") +``` + +#### Provided Tokenizers + + - `CharBPETokenizer`: The original BPE + - `ByteLevelBPETokenizer`: The byte level version of the BPE + - `SentencePieceBPETokenizer`: A BPE implementation compatible with the one used by SentencePiece + - `BertWordPieceTokenizer`: The famous Bert tokenizer, using WordPiece + +All of these can be used and trained as explained above! + +### Build your own + +Whenever these provided tokenizers don't give you enough freedom, you can build your own tokenizer, +by putting all the different parts you need together. +You can check how we implemented the [provided tokenizers](https://github.com/huggingface/tokenizers/tree/master/bindings/python/py_src/tokenizers/implementations) and adapt them easily to your own needs. + +#### Building a byte-level BPE + +Here is an example showing how to build your own byte-level BPE by putting all the different pieces +together, and then saving it to a single file: + +```python +from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, processors + +# Initialize a tokenizer +tokenizer = Tokenizer(models.BPE()) + +# Customize pre-tokenization and decoding +tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True) +tokenizer.decoder = decoders.ByteLevel() +tokenizer.post_processor = processors.ByteLevel(trim_offsets=True) + +# And then train +trainer = trainers.BpeTrainer( + vocab_size=20000, + min_frequency=2, + initial_alphabet=pre_tokenizers.ByteLevel.alphabet() +) +tokenizer.train([ + "./path/to/dataset/1.txt", + "./path/to/dataset/2.txt", + "./path/to/dataset/3.txt" +], trainer=trainer) + +# And Save it +tokenizer.save("byte-level-bpe.tokenizer.json", pretty=True) +``` + +Now, when you want to use this tokenizer, this is as simple as: + +```python +from tokenizers import Tokenizer + +tokenizer = Tokenizer.from_file("byte-level-bpe.tokenizer.json") + +encoded = tokenizer.encode("I can feel the magic, can you?") +``` + diff --git a/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/RECORD b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..ec9e5965bb165ef46304bd92e03a653bd9550f95 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/RECORD @@ -0,0 +1,45 @@ +tokenizers-0.21.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +tokenizers-0.21.0.dist-info/METADATA,sha256=oNldYkLKpnavqOq1XABp8c_yNaR65mGu_qaFlD0St2M,6719 +tokenizers-0.21.0.dist-info/RECORD,, +tokenizers-0.21.0.dist-info/WHEEL,sha256=PuLiPGpD0eVcoUkb9lueobt7VbCYShlDtLaTRPpT7Z0,127 +tokenizers/__init__.py,sha256=ZE5ZagUvobBScrHBQdEobhx4wqM0bsq9F9aLYkBNjYQ,2615 +tokenizers/__init__.pyi,sha256=jw34WZXaYu8NBBJ2_cypfOqJYxI7CXKPzlveisXw4XQ,40182 +tokenizers/__pycache__/__init__.cpython-311.pyc,, +tokenizers/decoders/__init__.py,sha256=hfwM6CFUDvlMGGL4-xsaaYz81K9P5rQI5ZL5UHWK8Y4,372 +tokenizers/decoders/__init__.pyi,sha256=U0dfPVxoGpb-RmNKzZMZebe0fK2riRMbxQh9yJMHjYE,7378 +tokenizers/decoders/__pycache__/__init__.cpython-311.pyc,, +tokenizers/implementations/__init__.py,sha256=VzAsplaIo7rl4AFO8Miu7ig7MfZjvonwVblZw01zR6M,310 +tokenizers/implementations/__pycache__/__init__.cpython-311.pyc,, +tokenizers/implementations/__pycache__/base_tokenizer.cpython-311.pyc,, +tokenizers/implementations/__pycache__/bert_wordpiece.cpython-311.pyc,, +tokenizers/implementations/__pycache__/byte_level_bpe.cpython-311.pyc,, +tokenizers/implementations/__pycache__/char_level_bpe.cpython-311.pyc,, +tokenizers/implementations/__pycache__/sentencepiece_bpe.cpython-311.pyc,, +tokenizers/implementations/__pycache__/sentencepiece_unigram.cpython-311.pyc,, +tokenizers/implementations/base_tokenizer.py,sha256=2TFZhLupaJiMDYGJuUNmxYJv-cnR8bDHmbMzaYpFROs,14206 +tokenizers/implementations/bert_wordpiece.py,sha256=sKCum0FKPYdSgJFJN8LDerVBoTDRSqyqSdrcm-lvQqI,5520 +tokenizers/implementations/byte_level_bpe.py,sha256=OA_jyy3EQmYTa6hnf-EKwLOFuyroqFYOJz25ysM2BUk,4289 +tokenizers/implementations/char_level_bpe.py,sha256=Q2ZEAW0xMQHF7YCUtmplwaxbU-J0P2NK4PJGMxUb-_c,5466 +tokenizers/implementations/sentencepiece_bpe.py,sha256=LwrofoohnUfME2lK2lQYoyQIhP84RP0CIlHRaj0hyNs,3738 +tokenizers/implementations/sentencepiece_unigram.py,sha256=SYiVXL8ZtqLXKpuqwnwmrfxgGotu8yAkOu7dLztEXIo,7580 +tokenizers/models/__init__.py,sha256=eJZ4HTAQZpxnKILNylWaTFqxXy-Ba6OKswWN47feeV8,176 +tokenizers/models/__init__.pyi,sha256=clPTwiyjz7FlVdEuwo_3Wa_TmQrbZhW0SGmnNylepnY,16929 +tokenizers/models/__pycache__/__init__.cpython-311.pyc,, +tokenizers/normalizers/__init__.py,sha256=_06w4cqRItveEgIddYaLMScgkSOkIAMIzYCesb5AA4U,841 +tokenizers/normalizers/__init__.pyi,sha256=dwfVsvg0YbeYoAaBSmKsImqL-tyxiDyHaaTFsZK4aZw,20897 +tokenizers/normalizers/__pycache__/__init__.cpython-311.pyc,, +tokenizers/pre_tokenizers/__init__.py,sha256=wd6KYQA_RsGSQK-HeG9opTRhv4ttSRkyno2dk6az-PM,557 +tokenizers/pre_tokenizers/__init__.pyi,sha256=dLtaxOgcBa85vQC6byvfKGCOWTEi4c42IcqimfatksQ,23602 +tokenizers/pre_tokenizers/__pycache__/__init__.cpython-311.pyc,, +tokenizers/processors/__init__.py,sha256=xM2DEKwKtHIumHsszM8AMkq-AlaqvBZFXWgLU8SNhOY,307 +tokenizers/processors/__init__.pyi,sha256=hx767ZY8SHhxb_hiXPRxm-f_KcoR4XDx7vfK2c0lR-Q,11357 +tokenizers/processors/__pycache__/__init__.cpython-311.pyc,, +tokenizers/tokenizers.abi3.so,sha256=uPe1mVjvIUFcXkPyyik0lcgOjPT3LKlTtAOTuDhZAN0,8942016 +tokenizers/tools/__init__.py,sha256=xG8caB9OHC8cbB01S5vYV14HZxhO6eWbLehsb70ppio,55 +tokenizers/tools/__pycache__/__init__.cpython-311.pyc,, +tokenizers/tools/__pycache__/visualizer.cpython-311.pyc,, +tokenizers/tools/visualizer-styles.css,sha256=zAydq1oGWD8QEll4-eyL8Llw0B1sty_hpIE3tYxL02k,4850 +tokenizers/tools/visualizer.py,sha256=gi-E2NCP7FuG6ujpQOdalSTXUlaV85V6NI-ZPPTvA_4,14625 +tokenizers/trainers/__init__.py,sha256=UTu22AGcp76IvpW45xLRbJWET04NxPW6NfCb2YYz0EM,248 +tokenizers/trainers/__init__.pyi,sha256=3TwFKts4me7zQfVRcSTmtXYiP4XwcRjfAYtwqoZVtoQ,5382 +tokenizers/trainers/__pycache__/__init__.cpython-311.pyc,, diff --git a/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/WHEEL b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..6ba72df3e2b4ae9dea5fd558e433705ddf053668 --- /dev/null +++ b/.venv/lib/python3.11/site-packages/tokenizers-0.21.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: maturin (1.7.5) +Root-Is-Purelib: false +Tag: cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64