diff --git a/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/LICENSE b/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e06d2081865a766a8668acc12878f98b27fc9ea0 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/NOTICE b/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..d134f281ef732a3697f32b24b42ce8c4c07f9019 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles-25.1.0.dist-info/licenses/NOTICE @@ -0,0 +1,2 @@ +Asyncio support for files +Copyright 2016 Tin Tvrtkovic diff --git a/py311/lib/python3.11/site-packages/aiofiles/tempfile/__init__.py b/py311/lib/python3.11/site-packages/aiofiles/tempfile/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b1c32c8018300a17bb8cb30f5500eaff84dd76bf --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/tempfile/__init__.py @@ -0,0 +1,357 @@ +import asyncio +import sys +from functools import partial, singledispatch +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOBase +from tempfile import NamedTemporaryFile as syncNamedTemporaryFile +from tempfile import SpooledTemporaryFile as syncSpooledTemporaryFile +from tempfile import TemporaryDirectory as syncTemporaryDirectory +from tempfile import TemporaryFile as syncTemporaryFile +from tempfile import _TemporaryFileWrapper as syncTemporaryFileWrapper + +from ..base import AiofilesContextManager +from ..threadpool.binary import AsyncBufferedIOBase, AsyncBufferedReader, AsyncFileIO +from ..threadpool.text import AsyncTextIOWrapper +from .temptypes import AsyncSpooledTemporaryFile, AsyncTemporaryDirectory + +__all__ = [ + "NamedTemporaryFile", + "TemporaryFile", + "SpooledTemporaryFile", + "TemporaryDirectory", +] + + +# ================================================================ +# Public methods for async open and return of temp file/directory +# objects with async interface +# ================================================================ +if sys.version_info >= (3, 12): + + def NamedTemporaryFile( + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + delete=True, + delete_on_close=True, + loop=None, + executor=None, + ): + """Async open a named temporary file""" + return AiofilesContextManager( + _temporary_file( + named=True, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + delete=delete, + delete_on_close=delete_on_close, + loop=loop, + executor=executor, + ) + ) + +else: + + def NamedTemporaryFile( + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + delete=True, + loop=None, + executor=None, + ): + """Async open a named temporary file""" + return AiofilesContextManager( + _temporary_file( + named=True, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + delete=delete, + loop=loop, + executor=executor, + ) + ) + + +def TemporaryFile( + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + loop=None, + executor=None, +): + """Async open an unnamed temporary file""" + return AiofilesContextManager( + _temporary_file( + named=False, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + loop=loop, + executor=executor, + ) + ) + + +def SpooledTemporaryFile( + max_size=0, + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + loop=None, + executor=None, +): + """Async open a spooled temporary file""" + return AiofilesContextManager( + _spooled_temporary_file( + max_size=max_size, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + loop=loop, + executor=executor, + ) + ) + + +def TemporaryDirectory(suffix=None, prefix=None, dir=None, loop=None, executor=None): + """Async open a temporary directory""" + return AiofilesContextManagerTempDir( + _temporary_directory( + suffix=suffix, prefix=prefix, dir=dir, loop=loop, executor=executor + ) + ) + + +# ========================================================= +# Internal coroutines to open new temp files/directories +# ========================================================= +if sys.version_info >= (3, 12): + + async def _temporary_file( + named=True, + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + delete=True, + delete_on_close=True, + loop=None, + executor=None, + max_size=0, + ): + """Async method to open a temporary file with async interface""" + if loop is None: + loop = asyncio.get_running_loop() + + if named: + cb = partial( + syncNamedTemporaryFile, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + delete=delete, + delete_on_close=delete_on_close, + ) + else: + cb = partial( + syncTemporaryFile, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + ) + + f = await loop.run_in_executor(executor, cb) + + # Wrap based on type of underlying IO object + if type(f) is syncTemporaryFileWrapper: + # _TemporaryFileWrapper was used (named files) + result = wrap(f.file, f, loop=loop, executor=executor) + result._closer = f._closer + return result + # IO object was returned directly without wrapper + return wrap(f, f, loop=loop, executor=executor) + +else: + + async def _temporary_file( + named=True, + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + delete=True, + loop=None, + executor=None, + max_size=0, + ): + """Async method to open a temporary file with async interface""" + if loop is None: + loop = asyncio.get_running_loop() + + if named: + cb = partial( + syncNamedTemporaryFile, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + delete=delete, + ) + else: + cb = partial( + syncTemporaryFile, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + ) + + f = await loop.run_in_executor(executor, cb) + + # Wrap based on type of underlying IO object + if type(f) is syncTemporaryFileWrapper: + # _TemporaryFileWrapper was used (named files) + result = wrap(f.file, f, loop=loop, executor=executor) + # add delete property + result.delete = f.delete + return result + # IO object was returned directly without wrapper + return wrap(f, f, loop=loop, executor=executor) + + +async def _spooled_temporary_file( + max_size=0, + mode="w+b", + buffering=-1, + encoding=None, + newline=None, + suffix=None, + prefix=None, + dir=None, + loop=None, + executor=None, +): + """Open a spooled temporary file with async interface""" + if loop is None: + loop = asyncio.get_running_loop() + + cb = partial( + syncSpooledTemporaryFile, + max_size=max_size, + mode=mode, + buffering=buffering, + encoding=encoding, + newline=newline, + suffix=suffix, + prefix=prefix, + dir=dir, + ) + + f = await loop.run_in_executor(executor, cb) + + # Single interface provided by SpooledTemporaryFile for all modes + return AsyncSpooledTemporaryFile(f, loop=loop, executor=executor) + + +async def _temporary_directory( + suffix=None, prefix=None, dir=None, loop=None, executor=None +): + """Async method to open a temporary directory with async interface""" + if loop is None: + loop = asyncio.get_running_loop() + + cb = partial(syncTemporaryDirectory, suffix, prefix, dir) + f = await loop.run_in_executor(executor, cb) + + return AsyncTemporaryDirectory(f, loop=loop, executor=executor) + + +class AiofilesContextManagerTempDir(AiofilesContextManager): + """With returns the directory location, not the object (matching sync lib)""" + + async def __aenter__(self): + self._obj = await self._coro + return self._obj.name + + +@singledispatch +def wrap(base_io_obj, file, *, loop=None, executor=None): + """Wrap the object with interface based on type of underlying IO""" + + msg = f"Unsupported IO type: {base_io_obj}" + raise TypeError(msg) + + +@wrap.register(TextIOBase) +def _(base_io_obj, file, *, loop=None, executor=None): + return AsyncTextIOWrapper(file, loop=loop, executor=executor) + + +@wrap.register(BufferedWriter) +def _(base_io_obj, file, *, loop=None, executor=None): + return AsyncBufferedIOBase(file, loop=loop, executor=executor) + + +@wrap.register(BufferedReader) +@wrap.register(BufferedRandom) +def _(base_io_obj, file, *, loop=None, executor=None): + return AsyncBufferedReader(file, loop=loop, executor=executor) + + +@wrap.register(FileIO) +def _(base_io_obj, file, *, loop=None, executor=None): + return AsyncFileIO(file, loop=loop, executor=executor) diff --git a/py311/lib/python3.11/site-packages/aiofiles/tempfile/temptypes.py b/py311/lib/python3.11/site-packages/aiofiles/tempfile/temptypes.py new file mode 100644 index 0000000000000000000000000000000000000000..8ae503276d65ce2a21dcae3dfebd5c715f5d5fca --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/tempfile/temptypes.py @@ -0,0 +1,70 @@ +"""Async wrappers for spooled temp files and temp directory objects""" + +from functools import partial + +from ..base import AsyncBase +from ..threadpool.utils import ( + cond_delegate_to_executor, + delegate_to_executor, + proxy_property_directly, +) + + +@delegate_to_executor("fileno", "rollover") +@cond_delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "readline", + "readlines", + "seek", + "tell", + "truncate", +) +@proxy_property_directly("closed", "encoding", "mode", "name", "newlines") +class AsyncSpooledTemporaryFile(AsyncBase): + """Async wrapper for SpooledTemporaryFile class""" + + async def _check(self): + if self._file._rolled: + return + max_size = self._file._max_size + if max_size and self._file.tell() > max_size: + await self.rollover() + + async def write(self, s): + """Implementation to anticipate rollover""" + if self._file._rolled: + cb = partial(self._file.write, s) + return await self._loop.run_in_executor(self._executor, cb) + + file = self._file._file # reference underlying base IO object + rv = file.write(s) + await self._check() + return rv + + async def writelines(self, iterable): + """Implementation to anticipate rollover""" + if self._file._rolled: + cb = partial(self._file.writelines, iterable) + return await self._loop.run_in_executor(self._executor, cb) + + file = self._file._file # reference underlying base IO object + rv = file.writelines(iterable) + await self._check() + return rv + + +@delegate_to_executor("cleanup") +@proxy_property_directly("name") +class AsyncTemporaryDirectory: + """Async wrapper for TemporaryDirectory class""" + + def __init__(self, file, loop, executor): + self._file = file + self._loop = loop + self._executor = executor + + async def close(self): + await self.cleanup() diff --git a/py311/lib/python3.11/site-packages/aiofiles/threadpool/__init__.py b/py311/lib/python3.11/site-packages/aiofiles/threadpool/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8054034d2bba3bbe24485a8b94e868dc58e94b82 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/threadpool/__init__.py @@ -0,0 +1,141 @@ +"""Handle files using a thread pool executor.""" + +import asyncio +import sys +from functools import partial, singledispatch +from io import ( + BufferedIOBase, + BufferedRandom, + BufferedReader, + BufferedWriter, + FileIO, + TextIOBase, +) + +from ..base import AiofilesContextManager +from .binary import ( + AsyncBufferedIOBase, + AsyncBufferedReader, + AsyncFileIO, + AsyncIndirectBufferedIOBase, +) +from .text import AsyncTextIndirectIOWrapper, AsyncTextIOWrapper + +sync_open = open + +__all__ = ( + "open", + "stdin", + "stdout", + "stderr", + "stdin_bytes", + "stdout_bytes", + "stderr_bytes", +) + + +def open( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + *, + loop=None, + executor=None, +): + return AiofilesContextManager( + _open( + file, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + closefd=closefd, + opener=opener, + loop=loop, + executor=executor, + ) + ) + + +async def _open( + file, + mode="r", + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None, + *, + loop=None, + executor=None, +): + """Open an asyncio file.""" + if loop is None: + loop = asyncio.get_running_loop() + cb = partial( + sync_open, + file, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + closefd=closefd, + opener=opener, + ) + f = await loop.run_in_executor(executor, cb) + + return wrap(f, loop=loop, executor=executor) + + +@singledispatch +def wrap(file, *, loop=None, executor=None): + msg = f"Unsupported io type: {file}." + raise TypeError(msg) + + +@wrap.register(TextIOBase) +def _(file, *, loop=None, executor=None): + return AsyncTextIOWrapper(file, loop=loop, executor=executor) + + +@wrap.register(BufferedWriter) +@wrap.register(BufferedIOBase) +def _(file, *, loop=None, executor=None): + return AsyncBufferedIOBase(file, loop=loop, executor=executor) + + +@wrap.register(BufferedReader) +@wrap.register(BufferedRandom) +def _(file, *, loop=None, executor=None): + return AsyncBufferedReader(file, loop=loop, executor=executor) + + +@wrap.register(FileIO) +def _(file, *, loop=None, executor=None): + return AsyncFileIO(file, loop=loop, executor=executor) + + +stdin = AsyncTextIndirectIOWrapper("sys.stdin", None, None, indirect=lambda: sys.stdin) +stdout = AsyncTextIndirectIOWrapper( + "sys.stdout", None, None, indirect=lambda: sys.stdout +) +stderr = AsyncTextIndirectIOWrapper( + "sys.stderr", None, None, indirect=lambda: sys.stderr +) +stdin_bytes = AsyncIndirectBufferedIOBase( + "sys.stdin.buffer", None, None, indirect=lambda: sys.stdin.buffer +) +stdout_bytes = AsyncIndirectBufferedIOBase( + "sys.stdout.buffer", None, None, indirect=lambda: sys.stdout.buffer +) +stderr_bytes = AsyncIndirectBufferedIOBase( + "sys.stderr.buffer", None, None, indirect=lambda: sys.stderr.buffer +) diff --git a/py311/lib/python3.11/site-packages/aiofiles/threadpool/binary.py b/py311/lib/python3.11/site-packages/aiofiles/threadpool/binary.py new file mode 100644 index 0000000000000000000000000000000000000000..63fcaff25959472c3282674a0c9e95160a8210b7 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/threadpool/binary.py @@ -0,0 +1,104 @@ +from ..base import AsyncBase, AsyncIndirectBase +from .utils import delegate_to_executor, proxy_method_directly, proxy_property_directly + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "read1", + "readinto", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "writable", + "write", + "writelines", +) +@proxy_method_directly("detach", "fileno", "readable") +@proxy_property_directly("closed", "raw", "name", "mode") +class AsyncBufferedIOBase(AsyncBase): + """The asyncio executor version of io.BufferedWriter and BufferedIOBase.""" + + +@delegate_to_executor("peek") +class AsyncBufferedReader(AsyncBufferedIOBase): + """The asyncio executor version of io.BufferedReader and Random.""" + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "readall", + "readinto", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "writable", + "write", + "writelines", +) +@proxy_method_directly("fileno", "readable") +@proxy_property_directly("closed", "name", "mode") +class AsyncFileIO(AsyncBase): + """The asyncio executor version of io.FileIO.""" + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "read1", + "readinto", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "writable", + "write", + "writelines", +) +@proxy_method_directly("detach", "fileno", "readable") +@proxy_property_directly("closed", "raw", "name", "mode") +class AsyncIndirectBufferedIOBase(AsyncIndirectBase): + """The indirect asyncio executor version of io.BufferedWriter and BufferedIOBase.""" + + +@delegate_to_executor("peek") +class AsyncIndirectBufferedReader(AsyncIndirectBufferedIOBase): + """The indirect asyncio executor version of io.BufferedReader and Random.""" + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "readall", + "readinto", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "writable", + "write", + "writelines", +) +@proxy_method_directly("fileno", "readable") +@proxy_property_directly("closed", "name", "mode") +class AsyncIndirectFileIO(AsyncIndirectBase): + """The indirect asyncio executor version of io.FileIO.""" diff --git a/py311/lib/python3.11/site-packages/aiofiles/threadpool/text.py b/py311/lib/python3.11/site-packages/aiofiles/threadpool/text.py new file mode 100644 index 0000000000000000000000000000000000000000..0e625909b6c960ebed4a0ed99941b28156fbf2d1 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/threadpool/text.py @@ -0,0 +1,64 @@ +from ..base import AsyncBase, AsyncIndirectBase +from .utils import delegate_to_executor, proxy_method_directly, proxy_property_directly + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "readable", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "write", + "writable", + "writelines", +) +@proxy_method_directly("detach", "fileno", "readable") +@proxy_property_directly( + "buffer", + "closed", + "encoding", + "errors", + "line_buffering", + "newlines", + "name", + "mode", +) +class AsyncTextIOWrapper(AsyncBase): + """The asyncio executor version of io.TextIOWrapper.""" + + +@delegate_to_executor( + "close", + "flush", + "isatty", + "read", + "readable", + "readline", + "readlines", + "seek", + "seekable", + "tell", + "truncate", + "write", + "writable", + "writelines", +) +@proxy_method_directly("detach", "fileno", "readable") +@proxy_property_directly( + "buffer", + "closed", + "encoding", + "errors", + "line_buffering", + "newlines", + "name", + "mode", +) +class AsyncTextIndirectIOWrapper(AsyncIndirectBase): + """The indirect asyncio executor version of io.TextIOWrapper.""" diff --git a/py311/lib/python3.11/site-packages/aiofiles/threadpool/utils.py b/py311/lib/python3.11/site-packages/aiofiles/threadpool/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fd9767a8aea444343b030f200fc04268c5a35182 --- /dev/null +++ b/py311/lib/python3.11/site-packages/aiofiles/threadpool/utils.py @@ -0,0 +1,71 @@ +import functools + + +def delegate_to_executor(*attrs): + def cls_builder(cls): + for attr_name in attrs: + setattr(cls, attr_name, _make_delegate_method(attr_name)) + return cls + + return cls_builder + + +def proxy_method_directly(*attrs): + def cls_builder(cls): + for attr_name in attrs: + setattr(cls, attr_name, _make_proxy_method(attr_name)) + return cls + + return cls_builder + + +def proxy_property_directly(*attrs): + def cls_builder(cls): + for attr_name in attrs: + setattr(cls, attr_name, _make_proxy_property(attr_name)) + return cls + + return cls_builder + + +def cond_delegate_to_executor(*attrs): + def cls_builder(cls): + for attr_name in attrs: + setattr(cls, attr_name, _make_cond_delegate_method(attr_name)) + return cls + + return cls_builder + + +def _make_delegate_method(attr_name): + async def method(self, *args, **kwargs): + cb = functools.partial(getattr(self._file, attr_name), *args, **kwargs) + return await self._loop.run_in_executor(self._executor, cb) + + return method + + +def _make_proxy_method(attr_name): + def method(self, *args, **kwargs): + return getattr(self._file, attr_name)(*args, **kwargs) + + return method + + +def _make_proxy_property(attr_name): + def proxy_property(self): + return getattr(self._file, attr_name) + + return property(proxy_property) + + +def _make_cond_delegate_method(attr_name): + """For spooled temp files, delegate only if rolled to file object""" + + async def method(self, *args, **kwargs): + if self._file._rolled: + cb = functools.partial(getattr(self._file, attr_name), *args, **kwargs) + return await self._loop.run_in_executor(self._executor, cb) + return getattr(self._file, attr_name)(*args, **kwargs) + + return method diff --git a/py311/lib/python3.11/site-packages/anyio-4.12.1.dist-info/licenses/LICENSE b/py311/lib/python3.11/site-packages/anyio-4.12.1.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..104eebf5a3002fccdaceef3a4cb936173c1c2035 --- /dev/null +++ b/py311/lib/python3.11/site-packages/anyio-4.12.1.dist-info/licenses/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2018 Alex Grönholm + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/__init__.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..af390484528d87bfb78377b228b47480d2284ea5 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""The pgen2 package.""" diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..25dd128fe7936aad60aa4a5119198a3edefb72ec Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.py new file mode 100644 index 0000000000000000000000000000000000000000..b936465f1a8739468a688d95106c153b2aca1e2e --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/conv.py @@ -0,0 +1,256 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# mypy: ignore-errors + +"""Convert graminit.[ch] spit out by pgen to Python code. + +Pgen is the Python parser generator. It is useful to quickly create a +parser from a grammar file in Python's grammar notation. But I don't +want my parsers to be written in C (yet), so I'm translating the +parsing tables to Python data structures and writing a Python parse +engine. + +Note that the token numbers are constants determined by the standard +Python tokenizer. The standard token module defines these numbers and +their names (the names are not used much). The token numbers are +hardcoded into the Python tokenizer and into pgen. A Python +implementation of the Python tokenizer is also available, in the +standard tokenize module. + +On the other hand, symbol numbers (representing the grammar's +non-terminals) are assigned by pgen based on the actual grammar +input. + +Note: this module is pretty much obsolete; the pgen module generates +equivalent grammar tables directly from the Grammar.txt input file +without having to invoke the Python pgen C program. + +""" + +# Python imports +import re + +# Local imports +from blib2to3.pgen2 import grammar, token + + +class Converter(grammar.Grammar): + """Grammar subclass that reads classic pgen output files. + + The run() method reads the tables as produced by the pgen parser + generator, typically contained in two C files, graminit.h and + graminit.c. The other methods are for internal use only. + + See the base class for more documentation. + + """ + + def run(self, graminit_h, graminit_c): + """Load the grammar tables from the text files written by pgen.""" + self.parse_graminit_h(graminit_h) + self.parse_graminit_c(graminit_c) + self.finish_off() + + def parse_graminit_h(self, filename): + """Parse the .h file written by pgen. (Internal) + + This file is a sequence of #define statements defining the + nonterminals of the grammar as numbers. We build two tables + mapping the numbers to names and back. + + """ + try: + f = open(filename) + except OSError as err: + print(f"Can't open {filename}: {err}") + return False + self.symbol2number = {} + self.number2symbol = {} + lineno = 0 + for line in f: + lineno += 1 + mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) + if not mo and line.strip(): + print(f"{filename}({lineno}): can't parse {line.strip()}") + else: + symbol, number = mo.groups() + number = int(number) + assert symbol not in self.symbol2number + assert number not in self.number2symbol + self.symbol2number[symbol] = number + self.number2symbol[number] = symbol + return True + + def parse_graminit_c(self, filename): + """Parse the .c file written by pgen. (Internal) + + The file looks as follows. The first two lines are always this: + + #include "pgenheaders.h" + #include "grammar.h" + + After that come four blocks: + + 1) one or more state definitions + 2) a table defining dfas + 3) a table defining labels + 4) a struct defining the grammar + + A state definition has the following form: + - one or more arc arrays, each of the form: + static arc arcs__[] = { + {, }, + ... + }; + - followed by a state array, of the form: + static state states_[] = { + {, arcs__}, + ... + }; + + """ + try: + f = open(filename) + except OSError as err: + print(f"Can't open {filename}: {err}") + return False + # The code below essentially uses f's iterator-ness! + lineno = 0 + + # Expect the two #include lines + lineno, line = lineno + 1, next(f) + assert line == '#include "pgenheaders.h"\n', (lineno, line) + lineno, line = lineno + 1, next(f) + assert line == '#include "grammar.h"\n', (lineno, line) + + # Parse the state definitions + lineno, line = lineno + 1, next(f) + allarcs = {} + states = [] + while line.startswith("static arc "): + while line.startswith("static arc "): + mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", line) + assert mo, (lineno, line) + n, m, k = list(map(int, mo.groups())) + arcs = [] + for _ in range(k): + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), (\d+)},$", line) + assert mo, (lineno, line) + i, j = list(map(int, mo.groups())) + arcs.append((i, j)) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + allarcs[(n, m)] = arcs + lineno, line = lineno + 1, next(f) + mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line) + assert mo, (lineno, line) + s, t = list(map(int, mo.groups())) + assert s == len(states), (lineno, line) + state = [] + for _ in range(t): + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line) + assert mo, (lineno, line) + k, n, m = list(map(int, mo.groups())) + arcs = allarcs[n, m] + assert k == len(arcs), (lineno, line) + state.append(arcs) + states.append(state) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + lineno, line = lineno + 1, next(f) + self.states = states + + # Parse the dfas + dfas = {} + mo = re.match(r"static dfa dfas\[(\d+)\] = {$", line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + for i in range(ndfas): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line) + assert mo, (lineno, line) + symbol = mo.group(2) + number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) + assert self.symbol2number[symbol] == number, (lineno, line) + assert self.number2symbol[number] == symbol, (lineno, line) + assert x == 0, (lineno, line) + state = states[z] + assert y == len(state), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) + assert mo, (lineno, line) + first = {} + rawbitset = eval(mo.group(1)) + for i, c in enumerate(rawbitset): + byte = ord(c) + for j in range(8): + if byte & (1 << j): + first[i * 8 + j] = 1 + dfas[number] = (state, first) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + self.dfas = dfas + + # Parse the labels + labels = [] + lineno, line = lineno + 1, next(f) + mo = re.match(r"static label labels\[(\d+)\] = {$", line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + for i in range(nlabels): + lineno, line = lineno + 1, next(f) + mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line) + assert mo, (lineno, line) + x, y = mo.groups() + x = int(x) + if y == "0": + y = None + else: + y = eval(y) + labels.append((x, y)) + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + self.labels = labels + + # Parse the grammar struct + lineno, line = lineno + 1, next(f) + assert line == "grammar _PyParser_Grammar = {\n", (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+(\d+),$", line) + assert mo, (lineno, line) + ndfas = int(mo.group(1)) + assert ndfas == len(self.dfas) + lineno, line = lineno + 1, next(f) + assert line == "\tdfas,\n", (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+{(\d+), labels},$", line) + assert mo, (lineno, line) + nlabels = int(mo.group(1)) + assert nlabels == len(self.labels), (lineno, line) + lineno, line = lineno + 1, next(f) + mo = re.match(r"\s+(\d+)$", line) + assert mo, (lineno, line) + start = int(mo.group(1)) + assert start in self.number2symbol, (lineno, line) + self.start = start + lineno, line = lineno + 1, next(f) + assert line == "};\n", (lineno, line) + try: + lineno, line = lineno + 1, next(f) + except StopIteration: + pass + else: + assert 0, (lineno, line) + + def finish_off(self): + """Create additional useful structures. (Internal).""" + self.keywords = {} # map from keyword strings to arc labels + self.tokens = {} # map from numeric token values to arc labels + for ilabel, (type, value) in enumerate(self.labels): + if type == token.NAME and value is not None: + self.keywords[value] = ilabel + elif value is None: + self.tokens[type] = ilabel diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..a117f3e90e28cbd4860989a577391d3e411c3ba4 Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.py new file mode 100644 index 0000000000000000000000000000000000000000..924c4bdca1e161f794a6481949094d56500ca621 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/driver.py @@ -0,0 +1,313 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +# Modifications: +# Copyright 2006 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Parser driver. + +This provides a high-level interface to parse a file into a syntax tree. + +""" + +__author__ = "Guido van Rossum " + +__all__ = ["Driver", "load_grammar"] + +# Python imports +import io +import logging +import os +import pkgutil +import sys +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from logging import Logger +from typing import Any, Union, cast + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.tokenize import TokenInfo +from blib2to3.pytree import NL + +# Pgen imports +from . import grammar, parse, pgen, token, tokenize + +Path = Union[str, "os.PathLike[str]"] + + +@dataclass +class ReleaseRange: + start: int + end: int | None = None + tokens: list[Any] = field(default_factory=list) + + def lock(self) -> None: + total_eaten = len(self.tokens) + self.end = self.start + total_eaten + + +class TokenProxy: + def __init__(self, generator: Any) -> None: + self._tokens = generator + self._counter = 0 + self._release_ranges: list[ReleaseRange] = [] + + @contextmanager + def release(self) -> Iterator["TokenProxy"]: + release_range = ReleaseRange(self._counter) + self._release_ranges.append(release_range) + try: + yield self + finally: + # Lock the last release range to the final position that + # has been eaten. + release_range.lock() + + def eat(self, point: int) -> Any: + eaten_tokens = self._release_ranges[-1].tokens + if point < len(eaten_tokens): + return eaten_tokens[point] + else: + while point >= len(eaten_tokens): + token = next(self._tokens) + eaten_tokens.append(token) + return token + + def __iter__(self) -> "TokenProxy": + return self + + def __next__(self) -> Any: + # If the current position is already compromised (looked up) + # return the eaten token, if not just go further on the given + # token producer. + for release_range in self._release_ranges: + assert release_range.end is not None + + start, end = release_range.start, release_range.end + if start <= self._counter < end: + token = release_range.tokens[self._counter - start] + break + else: + token = next(self._tokens) + self._counter += 1 + return token + + def can_advance(self, to: int) -> bool: + # Try to eat, fail if it can't. The eat operation is cached + # so there won't be any additional cost of eating here + try: + self.eat(to) + except StopIteration: + return False + else: + return True + + +class Driver: + def __init__(self, grammar: Grammar, logger: Logger | None = None) -> None: + self.grammar = grammar + if logger is None: + logger = logging.getLogger(__name__) + self.logger = logger + + def parse_tokens(self, tokens: Iterable[TokenInfo], debug: bool = False) -> NL: + """Parse a series of tokens and return the syntax tree.""" + # XXX Move the prefix computation into a wrapper around tokenize. + proxy = TokenProxy(tokens) + + p = parse.Parser(self.grammar) + p.setup(proxy=proxy) + + lineno = 1 + column = 0 + indent_columns: list[int] = [] + type = value = start = end = line_text = None + prefix = "" + + for quintuple in proxy: + type, value, start, end, line_text = quintuple + if start != (lineno, column): + assert (lineno, column) <= start, ((lineno, column), start) + s_lineno, s_column = start + if lineno < s_lineno: + prefix += "\n" * (s_lineno - lineno) + lineno = s_lineno + column = 0 + if column < s_column: + prefix += line_text[column:s_column] + column = s_column + if type in (tokenize.COMMENT, tokenize.NL): + prefix += value + lineno, column = end + if value.endswith("\n"): + lineno += 1 + column = 0 + continue + if type == token.OP: + type = grammar.opmap[value] + if debug: + assert type is not None + self.logger.debug( + "%s %r (prefix=%r)", token.tok_name[type], value, prefix + ) + if type == token.INDENT: + indent_columns.append(len(value)) + _prefix = prefix + value + prefix = "" + value = "" + elif type == token.DEDENT: + _indent_col = indent_columns.pop() + prefix, _prefix = self._partially_consume_prefix(prefix, _indent_col) + if p.addtoken(cast(int, type), value, (prefix, start)): + if debug: + self.logger.debug("Stop.") + break + prefix = "" + if type in {token.INDENT, token.DEDENT}: + prefix = _prefix + lineno, column = end + # FSTRING_MIDDLE and TSTRING_MIDDLE are the only token that can end with a + # newline, and `end` will point to the next line. For that case, don't + # increment lineno. + if value.endswith("\n") and type not in ( + token.FSTRING_MIDDLE, + token.TSTRING_MIDDLE, + ): + lineno += 1 + column = 0 + else: + # We never broke out -- EOF is too soon (how can this happen???) + assert start is not None + raise parse.ParseError("incomplete input", type, value, (prefix, start)) + assert p.rootnode is not None + return p.rootnode + + def parse_file( + self, filename: Path, encoding: str | None = None, debug: bool = False + ) -> NL: + """Parse a file and return the syntax tree.""" + with open(filename, encoding=encoding) as stream: + text = stream.read() + return self.parse_string(text, debug) + + def parse_string(self, text: str, debug: bool = False) -> NL: + """Parse a string and return the syntax tree.""" + tokens = tokenize.tokenize(text, grammar=self.grammar) + return self.parse_tokens(tokens, debug) + + def _partially_consume_prefix(self, prefix: str, column: int) -> tuple[str, str]: + lines: list[str] = [] + current_line = "" + current_column = 0 + wait_for_nl = False + for char in prefix: + current_line += char + if wait_for_nl: + if char == "\n": + if current_line.strip() and current_column < column: + res = "".join(lines) + return res, prefix[len(res) :] + + lines.append(current_line) + current_line = "" + current_column = 0 + wait_for_nl = False + elif char in " \t": + current_column += 1 + elif char == "\n": + # unexpected empty line + current_column = 0 + elif char == "\f": + current_column = 0 + else: + # indent is finished + wait_for_nl = True + return "".join(lines), current_line + + +def _generate_pickle_name(gt: Path, cache_dir: Path | None = None) -> str: + head, tail = os.path.splitext(gt) + if tail == ".txt": + tail = "" + name = head + tail + ".".join(map(str, sys.version_info)) + ".pickle" + if cache_dir: + return os.path.join(cache_dir, os.path.basename(name)) + else: + return name + + +def load_grammar( + gt: str = "Grammar.txt", + gp: str | None = None, + save: bool = True, + force: bool = False, + logger: Logger | None = None, +) -> Grammar: + """Load the grammar (maybe from a pickle).""" + if logger is None: + logger = logging.getLogger(__name__) + gp = _generate_pickle_name(gt) if gp is None else gp + if force or not _newer(gp, gt): + g: grammar.Grammar = pgen.generate_grammar(gt) + if save: + try: + g.dump(gp) + except OSError: + # Ignore error, caching is not vital. + pass + else: + g = grammar.Grammar() + g.load(gp) + return g + + +def _newer(a: str, b: str) -> bool: + """Inquire whether file a was written since file b.""" + if not os.path.exists(a): + return False + if not os.path.exists(b): + return True + return os.path.getmtime(a) >= os.path.getmtime(b) + + +def load_packaged_grammar( + package: str, grammar_source: str, cache_dir: Path | None = None +) -> grammar.Grammar: + """Normally, loads a pickled grammar by doing + pkgutil.get_data(package, pickled_grammar) + where *pickled_grammar* is computed from *grammar_source* by adding the + Python version and using a ``.pickle`` extension. + + However, if *grammar_source* is an extant file, load_grammar(grammar_source) + is called instead. This facilitates using a packaged grammar file when needed + but preserves load_grammar's automatic regeneration behavior when possible. + + """ + if os.path.isfile(grammar_source): + gp = _generate_pickle_name(grammar_source, cache_dir) if cache_dir else None + return load_grammar(grammar_source, gp=gp) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source), cache_dir) + data = pkgutil.get_data(package, pickled_name) + assert data is not None + g = grammar.Grammar() + g.loads(data) + return g + + +def main(*args: str) -> bool: + """Main program, when run as a script: produce grammar pickle files. + + Calls load_grammar for each argument, a path to a grammar text file. + """ + if not args: + args = tuple(sys.argv[1:]) + logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s") + for gt in args: + load_grammar(gt, save=True, force=True) + return True + + +if __name__ == "__main__": + sys.exit(int(not main())) diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..bf00fbf05c9dc493cb5055f93f4724fceea74feb Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.py new file mode 100644 index 0000000000000000000000000000000000000000..9cf240377545cca093602de360b09c25f0a49af9 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/grammar.py @@ -0,0 +1,228 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""This module defines the data structures used to represent a grammar. + +These are a bit arcane because they are derived from the data +structures used by Python's 'pgen' parser generator. + +There's also a table here mapping operators to their names in the +token module; the Python tokenize module reports all operators as the +fallback token code OP, but the parser needs the actual token code. + +""" + +# Python imports +import os +import pickle +import tempfile +from typing import Any, Optional, TypeVar, Union + +# Local imports +from . import token + +_P = TypeVar("_P", bound="Grammar") +Label = tuple[int, Optional[str]] +DFA = list[list[tuple[int, int]]] +DFAS = tuple[DFA, dict[int, int]] +Path = Union[str, "os.PathLike[str]"] + + +class Grammar: + """Pgen parsing tables conversion class. + + Once initialized, this class supplies the grammar tables for the + parsing engine implemented by parse.py. The parsing engine + accesses the instance variables directly. The class here does not + provide initialization of the tables; several subclasses exist to + do this (see the conv and pgen modules). + + The load() method reads the tables from a pickle file, which is + much faster than the other ways offered by subclasses. The pickle + file is written by calling dump() (after loading the grammar + tables using a subclass). The report() method prints a readable + representation of the tables to stdout, for debugging. + + The instance variables are as follows: + + symbol2number -- a dict mapping symbol names to numbers. Symbol + numbers are always 256 or higher, to distinguish + them from token numbers, which are between 0 and + 255 (inclusive). + + number2symbol -- a dict mapping numbers to symbol names; + these two are each other's inverse. + + states -- a list of DFAs, where each DFA is a list of + states, each state is a list of arcs, and each + arc is a (i, j) pair where i is a label and j is + a state number. The DFA number is the index into + this list. (This name is slightly confusing.) + Final states are represented by a special arc of + the form (0, j) where j is its own state number. + + dfas -- a dict mapping symbol numbers to (DFA, first) + pairs, where DFA is an item from the states list + above, and first is a set of tokens that can + begin this grammar rule (represented by a dict + whose values are always 1). + + labels -- a list of (x, y) pairs where x is either a token + number or a symbol number, and y is either None + or a string; the strings are keywords. The label + number is the index in this list; label numbers + are used to mark state transitions (arcs) in the + DFAs. + + start -- the number of the grammar's start symbol. + + keywords -- a dict mapping keyword strings to arc labels. + + tokens -- a dict mapping token numbers to arc labels. + + """ + + def __init__(self) -> None: + self.symbol2number: dict[str, int] = {} + self.number2symbol: dict[int, str] = {} + self.states: list[DFA] = [] + self.dfas: dict[int, DFAS] = {} + self.labels: list[Label] = [(0, "EMPTY")] + self.keywords: dict[str, int] = {} + self.soft_keywords: dict[str, int] = {} + self.tokens: dict[int, int] = {} + self.symbol2label: dict[str, int] = {} + self.version: tuple[int, int] = (0, 0) + self.start = 256 + # Python 3.7+ parses async as a keyword, not an identifier + self.async_keywords = False + + def dump(self, filename: Path) -> None: + """Dump the grammar tables to a pickle file.""" + + # mypyc generates objects that don't have a __dict__, but they + # do have __getstate__ methods that will return an equivalent + # dictionary + if hasattr(self, "__dict__"): + d = self.__dict__ + else: + d = self.__getstate__() # type: ignore + + with tempfile.NamedTemporaryFile( + dir=os.path.dirname(filename), delete=False + ) as f: + pickle.dump(d, f, pickle.HIGHEST_PROTOCOL) + os.replace(f.name, filename) + + def _update(self, attrs: dict[str, Any]) -> None: + for k, v in attrs.items(): + setattr(self, k, v) + + def load(self, filename: Path) -> None: + """Load the grammar tables from a pickle file.""" + with open(filename, "rb") as f: + d = pickle.load(f) + self._update(d) + + def loads(self, pkl: bytes) -> None: + """Load the grammar tables from a pickle bytes object.""" + self._update(pickle.loads(pkl)) + + def copy(self: _P) -> _P: + """ + Copy the grammar. + """ + new = self.__class__() + for dict_attr in ( + "symbol2number", + "number2symbol", + "dfas", + "keywords", + "soft_keywords", + "tokens", + "symbol2label", + ): + setattr(new, dict_attr, getattr(self, dict_attr).copy()) + new.labels = self.labels[:] + new.states = self.states[:] + new.start = self.start + new.version = self.version + new.async_keywords = self.async_keywords + return new + + def report(self) -> None: + """Dump the grammar tables to standard output, for debugging.""" + from pprint import pprint + + print("s2n") + pprint(self.symbol2number) + print("n2s") + pprint(self.number2symbol) + print("states") + pprint(self.states) + print("dfas") + pprint(self.dfas) + print("labels") + pprint(self.labels) + print("start", self.start) + + +# Map from operator to number (since tokenize doesn't do this) + +opmap_raw = """ +( LPAR +) RPAR +[ LSQB +] RSQB +: COLON +, COMMA +; SEMI ++ PLUS +- MINUS +* STAR +/ SLASH +| VBAR +& AMPER +< LESS +> GREATER += EQUAL +. DOT +% PERCENT +` BACKQUOTE +{ LBRACE +} RBRACE +@ AT +@= ATEQUAL +== EQEQUAL +!= NOTEQUAL +<> NOTEQUAL +<= LESSEQUAL +>= GREATEREQUAL +~ TILDE +^ CIRCUMFLEX +<< LEFTSHIFT +>> RIGHTSHIFT +** DOUBLESTAR ++= PLUSEQUAL +-= MINEQUAL +*= STAREQUAL +/= SLASHEQUAL +%= PERCENTEQUAL +&= AMPEREQUAL +|= VBAREQUAL +^= CIRCUMFLEXEQUAL +<<= LEFTSHIFTEQUAL +>>= RIGHTSHIFTEQUAL +**= DOUBLESTAREQUAL +// DOUBLESLASH +//= DOUBLESLASHEQUAL +-> RARROW +:= COLONEQUAL +! BANG +""" + +opmap = {} +for line in opmap_raw.splitlines(): + if line: + op, name = line.split() + opmap[op] = getattr(token, name) diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..9bed71e565197adc0760f62204e8edaa77a2fcd7 Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.py new file mode 100644 index 0000000000000000000000000000000000000000..a384c08dcecaf6bceb375247223050dca7b3a1ac --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/literals.py @@ -0,0 +1,65 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Safely evaluate Python string literals without using eval().""" + +import re + +simple_escapes: dict[str, str] = { + "a": "\a", + "b": "\b", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", + "'": "'", + '"': '"', + "\\": "\\", +} + + +def escape(m: re.Match[str]) -> str: + all, tail = m.group(0, 1) + assert all.startswith("\\") + esc = simple_escapes.get(tail) + if esc is not None: + return esc + if tail.startswith("x"): + hexes = tail[1:] + if len(hexes) < 2: + raise ValueError(f"invalid hex string escape ('\\{tail}')") + try: + i = int(hexes, 16) + except ValueError: + raise ValueError(f"invalid hex string escape ('\\{tail}')") from None + else: + try: + i = int(tail, 8) + except ValueError: + raise ValueError(f"invalid octal string escape ('\\{tail}')") from None + return chr(i) + + +def evalString(s: str) -> str: + assert s.startswith("'") or s.startswith('"'), repr(s[:1]) + q = s[0] + if s[:3] == q * 3: + q = q * 3 + assert s.endswith(q), repr(s[-len(q) :]) + assert len(s) >= 2 * len(q) + s = s[len(q) : -len(q)] + return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) + + +def test() -> None: + for i in range(256): + c = chr(i) + s = repr(c) + e = evalString(s) + if e != c: + print(i, c, s, e) + + +if __name__ == "__main__": + test() diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..08bce776b6f8ec74bd2af5885c94831ed67b7758 Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.py new file mode 100644 index 0000000000000000000000000000000000000000..4efce755c242c22e3792b242e7044841dd8e6e24 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/parse.py @@ -0,0 +1,395 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Parser engine for the grammar tables generated by pgen. + +The grammar table must be loaded first. + +See Parser/parser.c in the Python distribution for additional info on +how this parsing engine works. + +""" + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING, Union, cast + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert + +# Local imports +from . import grammar, token, tokenize + +if TYPE_CHECKING: + from blib2to3.pgen2.driver import TokenProxy + + +Results = dict[str, NL] +Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]] +DFA = list[list[tuple[int, int]]] +DFAS = tuple[DFA, dict[int, int]] + + +def lam_sub(grammar: Grammar, node: RawNode) -> NL: + assert node[3] is not None + return Node(type=node[0], children=node[3], context=node[2]) + + +# A placeholder node, used when parser is backtracking. +DUMMY_NODE = (-1, None, None, None) + + +def stack_copy( + stack: list[tuple[DFAS, int, RawNode]], +) -> list[tuple[DFAS, int, RawNode]]: + """Nodeless stack copy.""" + return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack] + + +class Recorder: + def __init__(self, parser: "Parser", ilabels: list[int], context: Context) -> None: + self.parser = parser + self._ilabels = ilabels + self.context = context # not really matter + + self._dead_ilabels: set[int] = set() + self._start_point = self.parser.stack + self._points = {ilabel: stack_copy(self._start_point) for ilabel in ilabels} + + @property + def ilabels(self) -> set[int]: + return self._dead_ilabels.symmetric_difference(self._ilabels) + + @contextmanager + def switch_to(self, ilabel: int) -> Iterator[None]: + with self.backtrack(): + self.parser.stack = self._points[ilabel] + try: + yield + except ParseError: + self._dead_ilabels.add(ilabel) + finally: + self.parser.stack = self._start_point + + @contextmanager + def backtrack(self) -> Iterator[None]: + """ + Use the node-level invariant ones for basic parsing operations (push/pop/shift). + These still will operate on the stack; but they won't create any new nodes, or + modify the contents of any other existing nodes. + + This saves us a ton of time when we are backtracking, since we + want to restore to the initial state as quick as possible, which + can only be done by having as little mutatations as possible. + """ + is_backtracking = self.parser.is_backtracking + try: + self.parser.is_backtracking = True + yield + finally: + self.parser.is_backtracking = is_backtracking + + def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: + for ilabel in self.ilabels: + with self.switch_to(ilabel): + if raw: + self.parser._addtoken(ilabel, tok_type, tok_val, self.context) + else: + self.parser.addtoken(tok_type, tok_val, self.context) + + def determine_route( + self, value: str | None = None, force: bool = False + ) -> int | None: + alive_ilabels = self.ilabels + if len(alive_ilabels) == 0: + *_, most_successful_ilabel = self._dead_ilabels + raise ParseError("bad input", most_successful_ilabel, value, self.context) + + ilabel, *rest = alive_ilabels + if force or not rest: + return ilabel + else: + return None + + +class ParseError(Exception): + """Exception to signal the parser is stuck.""" + + def __init__( + self, msg: str, type: int | None, value: str | None, context: Context + ) -> None: + Exception.__init__( + self, f"{msg}: type={type!r}, value={value!r}, context={context!r}" + ) + self.msg = msg + self.type = type + self.value = value + self.context = context + + +class Parser: + """Parser engine. + + The proper usage sequence is: + + p = Parser(grammar, [converter]) # create instance + p.setup([start]) # prepare for parsing + : + if p.addtoken(...): # parse a token; may raise ParseError + break + root = p.rootnode # root of abstract syntax tree + + A Parser instance may be reused by calling setup() repeatedly. + + A Parser instance contains state pertaining to the current token + sequence, and should not be used concurrently by different threads + to parse separate token sequences. + + See driver.py for how to get input tokens by tokenizing a file or + string. + + Parsing is complete when addtoken() returns True; the root of the + abstract syntax tree can then be retrieved from the rootnode + instance variable. When a syntax error occurs, addtoken() raises + the ParseError exception. There is no error recovery; the parser + cannot be used after a syntax error was reported (but it can be + reinitialized by calling setup()). + + """ + + def __init__(self, grammar: Grammar, convert: Convert | None = None) -> None: + """Constructor. + + The grammar argument is a grammar.Grammar instance; see the + grammar module for more information. + + The parser is not ready yet for parsing; you must call the + setup() method to get it started. + + The optional convert argument is a function mapping concrete + syntax tree nodes to abstract syntax tree nodes. If not + given, no conversion is done and the syntax tree produced is + the concrete syntax tree. If given, it must be a function of + two arguments, the first being the grammar (a grammar.Grammar + instance), and the second being the concrete syntax tree node + to be converted. The syntax tree is converted from the bottom + up. + + **post-note: the convert argument is ignored since for Black's + usage, convert will always be blib2to3.pytree.convert. Allowing + this to be dynamic hurts mypyc's ability to use early binding. + These docs are left for historical and informational value. + + A concrete syntax tree node is a (type, value, context, nodes) + tuple, where type is the node type (a token or symbol number), + value is None for symbols and a string for tokens, context is + None or an opaque value used for error reporting (typically a + (lineno, offset) pair), and nodes is a list of children for + symbols, and None for tokens. + + An abstract syntax tree node may be anything; this is entirely + up to the converter function. + + """ + self.grammar = grammar + # See note in docstring above. TL;DR this is ignored. + self.convert = convert or lam_sub + self.is_backtracking = False + self.last_token: int | None = None + + def setup(self, proxy: "TokenProxy", start: int | None = None) -> None: + """Prepare for parsing. + + This *must* be called before starting to parse. + + The optional argument is an alternative start symbol; it + defaults to the grammar's start symbol. + + You can use a Parser instance to parse any number of programs; + each time you call setup() the parser is reset to an initial + state determined by the (implicit or explicit) start symbol. + + """ + if start is None: + start = self.grammar.start + # Each stack entry is a tuple: (dfa, state, node). + # A node is a tuple: (type, value, context, children), + # where children is a list of nodes or None, and context may be None. + newnode: RawNode = (start, None, None, []) + stackentry = (self.grammar.dfas[start], 0, newnode) + self.stack: list[tuple[DFAS, int, RawNode]] = [stackentry] + self.rootnode: NL | None = None + self.used_names: set[str] = set() + self.proxy = proxy + self.last_token = None + + def addtoken(self, type: int, value: str, context: Context) -> bool: + """Add a token; return True iff this is the end of the program.""" + # Map from token to label + ilabels = self.classify(type, value, context) + assert len(ilabels) >= 1 + + # If we have only one state to advance, we'll directly + # take it as is. + if len(ilabels) == 1: + [ilabel] = ilabels + return self._addtoken(ilabel, type, value, context) + + # If there are multiple states which we can advance (only + # happen under soft-keywords), then we will try all of them + # in parallel and as soon as one state can reach further than + # the rest, we'll choose that one. This is a pretty hacky + # and hopefully temporary algorithm. + # + # For a more detailed explanation, check out this post: + # https://tree.science/what-the-backtracking.html + + with self.proxy.release() as proxy: + counter, force = 0, False + recorder = Recorder(self, ilabels, context) + recorder.add_token(type, value, raw=True) + + next_token_value = value + while recorder.determine_route(next_token_value) is None: + if not proxy.can_advance(counter): + force = True + break + + next_token_type, next_token_value, *_ = proxy.eat(counter) + if next_token_type in (tokenize.COMMENT, tokenize.NL): + counter += 1 + continue + + if next_token_type == tokenize.OP: + next_token_type = grammar.opmap[next_token_value] + + recorder.add_token(next_token_type, next_token_value) + counter += 1 + + ilabel = cast(int, recorder.determine_route(next_token_value, force=force)) + assert ilabel is not None + + return self._addtoken(ilabel, type, value, context) + + def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> bool: + # Loop until the token is shifted; may raise exceptions + while True: + dfa, state, node = self.stack[-1] + states, first = dfa + arcs = states[state] + # Look for a state with this label + for i, newstate in arcs: + t = self.grammar.labels[i][0] + if t >= 256: + # See if it's a symbol and if we're in its first set + itsdfa = self.grammar.dfas[t] + itsstates, itsfirst = itsdfa + if ilabel in itsfirst: + # Push a symbol + self.push(t, itsdfa, newstate, context) + break # To continue the outer while loop + + elif ilabel == i: + # Look it up in the list of labels + # Shift a token; we're done with it + self.shift(type, value, newstate, context) + # Pop while we are in an accept-only state + state = newstate + while states[state] == [(0, state)]: + self.pop() + if not self.stack: + # Done parsing! + return True + dfa, state, node = self.stack[-1] + states, first = dfa + # Done with this token + self.last_token = type + return False + + else: + if (0, state) in arcs: + # An accepting state, pop it and try something else + self.pop() + if not self.stack: + # Done parsing, but another token is input + raise ParseError("too much input", type, value, context) + else: + # No success finding a transition + raise ParseError("bad input", type, value, context) + + def classify(self, type: int, value: str, context: Context) -> list[int]: + """Turn a token into a label. (Internal) + + Depending on whether the value is a soft-keyword or not, + this function may return multiple labels to choose from.""" + if type == token.NAME: + # Keep a listing of all used names + self.used_names.add(value) + # Check for reserved words + if value in self.grammar.keywords: + return [self.grammar.keywords[value]] + elif value in self.grammar.soft_keywords: + assert type in self.grammar.tokens + # Current soft keywords (match, case, type) can only appear at the + # beginning of a statement. So as a shortcut, don't try to treat them + # like keywords in any other context. + # ('_' is also a soft keyword in the real grammar, but for our grammar + # it's just an expression, so we don't need to treat it specially.) + if self.last_token not in ( + None, + token.INDENT, + token.DEDENT, + token.NEWLINE, + token.SEMI, + token.COLON, + ): + return [self.grammar.tokens[type]] + return [ + self.grammar.tokens[type], + self.grammar.soft_keywords[value], + ] + + ilabel = self.grammar.tokens.get(type) + if ilabel is None: + raise ParseError("bad token", type, value, context) + return [ilabel] + + def shift(self, type: int, value: str, newstate: int, context: Context) -> None: + """Shift a token. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + else: + dfa, state, node = self.stack[-1] + rawnode: RawNode = (type, value, context, None) + newnode = convert(self.grammar, rawnode) + assert node[-1] is not None + node[-1].append(newnode) + self.stack[-1] = (dfa, newstate, node) + + def push(self, type: int, newdfa: DFAS, newstate: int, context: Context) -> None: + """Push a nonterminal. (Internal)""" + if self.is_backtracking: + dfa, state, _ = self.stack[-1] + self.stack[-1] = (dfa, newstate, DUMMY_NODE) + self.stack.append((newdfa, 0, DUMMY_NODE)) + else: + dfa, state, node = self.stack[-1] + newnode: RawNode = (type, None, context, []) + self.stack[-1] = (dfa, newstate, node) + self.stack.append((newdfa, 0, newnode)) + + def pop(self) -> None: + """Pop a nonterminal. (Internal)""" + if self.is_backtracking: + self.stack.pop() + else: + popdfa, popstate, popnode = self.stack.pop() + newnode = convert(self.grammar, popnode) + if self.stack: + dfa, state, node = self.stack[-1] + assert node[-1] is not None + node[-1].append(newnode) + else: + self.rootnode = newnode + self.rootnode.used_names = self.used_names diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..2ada35e2a78a5ada93def1ad4362bdf647571912 Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.py new file mode 100644 index 0000000000000000000000000000000000000000..7e6cdb480da5aabfa887e6da833e354feea96391 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/pgen.py @@ -0,0 +1,411 @@ +# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +import os +from collections.abc import Iterator, Sequence +from typing import IO, Any, NoReturn, Union + +from blib2to3.pgen2 import grammar, token, tokenize +from blib2to3.pgen2.tokenize import TokenInfo + +Path = Union[str, "os.PathLike[str]"] + + +class PgenGrammar(grammar.Grammar): + pass + + +class ParserGenerator: + filename: Path + stream: IO[str] + generator: Iterator[TokenInfo] + first: dict[str, dict[str, int] | None] + + def __init__(self, filename: Path, stream: IO[str] | None = None) -> None: + close_stream = None + if stream is None: + stream = open(filename, encoding="utf-8") + close_stream = stream.close + self.filename = filename + self.generator = tokenize.tokenize(stream.read()) + self.gettoken() # Initialize lookahead + self.dfas, self.startsymbol = self.parse() + if close_stream is not None: + close_stream() + self.first = {} # map from symbol name to set of tokens + self.addfirstsets() + + def make_grammar(self) -> PgenGrammar: + c = PgenGrammar() + names = list(self.dfas.keys()) + names.sort() + names.remove(self.startsymbol) + names.insert(0, self.startsymbol) + for name in names: + i = 256 + len(c.symbol2number) + c.symbol2number[name] = i + c.number2symbol[i] = name + for name in names: + dfa = self.dfas[name] + states = [] + for state in dfa: + arcs = [] + for label, next in sorted(state.arcs.items()): + arcs.append((self.make_label(c, label), dfa.index(next))) + if state.isfinal: + arcs.append((0, dfa.index(state))) + states.append(arcs) + c.states.append(states) + c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name)) + c.start = c.symbol2number[self.startsymbol] + return c + + def make_first(self, c: PgenGrammar, name: str) -> dict[int, int]: + rawfirst = self.first[name] + assert rawfirst is not None + first = {} + for label in sorted(rawfirst): + ilabel = self.make_label(c, label) + ##assert ilabel not in first # XXX failed on <> ... != + first[ilabel] = 1 + return first + + def make_label(self, c: PgenGrammar, label: str) -> int: + # XXX Maybe this should be a method on a subclass of converter? + ilabel = len(c.labels) + if label[0].isalpha(): + # Either a symbol name or a named token + if label in c.symbol2number: + # A symbol name (a non-terminal) + if label in c.symbol2label: + return c.symbol2label[label] + else: + c.labels.append((c.symbol2number[label], None)) + c.symbol2label[label] = ilabel + return ilabel + else: + # A named token (NAME, NUMBER, STRING) + itoken = getattr(token, label, None) + assert isinstance(itoken, int), label + assert itoken in token.tok_name, label + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + else: + # Either a keyword or an operator + assert label[0] in ('"', "'"), label + value = eval(label) + if value[0].isalpha(): + if label[0] == '"': + keywords = c.soft_keywords + else: + keywords = c.keywords + + # A keyword + if value in keywords: + return keywords[value] + else: + c.labels.append((token.NAME, value)) + keywords[value] = ilabel + return ilabel + else: + # An operator (any non-numeric token) + itoken = grammar.opmap[value] # Fails if unknown token + if itoken in c.tokens: + return c.tokens[itoken] + else: + c.labels.append((itoken, None)) + c.tokens[itoken] = ilabel + return ilabel + + def addfirstsets(self) -> None: + names = list(self.dfas.keys()) + names.sort() + for name in names: + if name not in self.first: + self.calcfirst(name) + # print name, self.first[name].keys() + + def calcfirst(self, name: str) -> None: + dfa = self.dfas[name] + self.first[name] = None # dummy to detect left recursion + state = dfa[0] + totalset: dict[str, int] = {} + overlapcheck = {} + for label in state.arcs: + if label in self.dfas: + if label in self.first: + fset = self.first[label] + if fset is None: + raise ValueError(f"recursion for rule {name!r}") + else: + self.calcfirst(label) + fset = self.first[label] + assert fset is not None + totalset.update(fset) + overlapcheck[label] = fset + else: + totalset[label] = 1 + overlapcheck[label] = {label: 1} + inverse: dict[str, str] = {} + for label, itsfirst in overlapcheck.items(): + for symbol in itsfirst: + if symbol in inverse: + raise ValueError( + f"rule {name} is ambiguous; {symbol} is in the first sets of" + f" {label} as well as {inverse[symbol]}" + ) + inverse[symbol] = label + self.first[name] = totalset + + def parse(self) -> tuple[dict[str, list["DFAState"]], str]: + dfas = {} + startsymbol: str | None = None + # MSTART: (NEWLINE | RULE)* ENDMARKER + while self.type != token.ENDMARKER: + while self.type == token.NEWLINE: + self.gettoken() + # RULE: NAME ':' RHS NEWLINE + name = self.expect(token.NAME) + self.expect(token.OP, ":") + a, z = self.parse_rhs() + self.expect(token.NEWLINE) + # self.dump_nfa(name, a, z) + dfa = self.make_dfa(a, z) + # self.dump_dfa(name, dfa) + # oldlen = len(dfa) + self.simplify_dfa(dfa) + # newlen = len(dfa) + dfas[name] = dfa + # print name, oldlen, newlen + if startsymbol is None: + startsymbol = name + assert startsymbol is not None + return dfas, startsymbol + + def make_dfa(self, start: "NFAState", finish: "NFAState") -> list["DFAState"]: + # To turn an NFA into a DFA, we define the states of the DFA + # to correspond to *sets* of states of the NFA. Then do some + # state reduction. Let's represent sets as dicts with 1 for + # values. + assert isinstance(start, NFAState) + assert isinstance(finish, NFAState) + + def closure(state: NFAState) -> dict[NFAState, int]: + base: dict[NFAState, int] = {} + addclosure(state, base) + return base + + def addclosure(state: NFAState, base: dict[NFAState, int]) -> None: + assert isinstance(state, NFAState) + if state in base: + return + base[state] = 1 + for label, next in state.arcs: + if label is None: + addclosure(next, base) + + states = [DFAState(closure(start), finish)] + for state in states: # NB states grows while we're iterating + arcs: dict[str, dict[NFAState, int]] = {} + for nfastate in state.nfaset: + for label, next in nfastate.arcs: + if label is not None: + addclosure(next, arcs.setdefault(label, {})) + for label, nfaset in sorted(arcs.items()): + for st in states: + if st.nfaset == nfaset: + break + else: + st = DFAState(nfaset, finish) + states.append(st) + state.addarc(st, label) + return states # List of DFAState instances; first one is start + + def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: + print("Dump of NFA for", name) + todo = [start] + for i, state in enumerate(todo): + print(" State", i, state is finish and "(final)" or "") + for label, next in state.arcs: + if next in todo: + j = todo.index(next) + else: + j = len(todo) + todo.append(next) + if label is None: + print(f" -> {j}") + else: + print(f" {label} -> {j}") + + def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: + print("Dump of DFA for", name) + for i, state in enumerate(dfa): + print(" State", i, state.isfinal and "(final)" or "") + for label, next in sorted(state.arcs.items()): + print(f" {label} -> {dfa.index(next)}") + + def simplify_dfa(self, dfa: list["DFAState"]) -> None: + # This is not theoretically optimal, but works well enough. + # Algorithm: repeatedly look for two states that have the same + # set of arcs (same labels pointing to the same nodes) and + # unify them, until things stop changing. + + # dfa is a list of DFAState instances + changes = True + while changes: + changes = False + for i, state_i in enumerate(dfa): + for j in range(i + 1, len(dfa)): + state_j = dfa[j] + if state_i == state_j: + # print " unify", i, j + del dfa[j] + for state in dfa: + state.unifystate(state_j, state_i) + changes = True + break + + def parse_rhs(self) -> tuple["NFAState", "NFAState"]: + # RHS: ALT ('|' ALT)* + a, z = self.parse_alt() + if self.value != "|": + return a, z + else: + aa = NFAState() + zz = NFAState() + aa.addarc(a) + z.addarc(zz) + while self.value == "|": + self.gettoken() + a, z = self.parse_alt() + aa.addarc(a) + z.addarc(zz) + return aa, zz + + def parse_alt(self) -> tuple["NFAState", "NFAState"]: + # ALT: ITEM+ + a, b = self.parse_item() + while self.value in ("(", "[") or self.type in (token.NAME, token.STRING): + c, d = self.parse_item() + b.addarc(c) + b = d + return a, b + + def parse_item(self) -> tuple["NFAState", "NFAState"]: + # ITEM: '[' RHS ']' | ATOM ['+' | '*'] + if self.value == "[": + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, "]") + a.addarc(z) + return a, z + else: + a, z = self.parse_atom() + value = self.value + if value not in ("+", "*"): + return a, z + self.gettoken() + z.addarc(a) + if value == "+": + return a, z + else: + return a, a + + def parse_atom(self) -> tuple["NFAState", "NFAState"]: + # ATOM: '(' RHS ')' | NAME | STRING + if self.value == "(": + self.gettoken() + a, z = self.parse_rhs() + self.expect(token.OP, ")") + return a, z + elif self.type in (token.NAME, token.STRING): + a = NFAState() + z = NFAState() + a.addarc(z, self.value) + self.gettoken() + return a, z + else: + self.raise_error( + f"expected (...) or NAME or STRING, got {self.type}/{self.value}" + ) + + def expect(self, type: int, value: Any | None = None) -> str: + if self.type != type or (value is not None and self.value != value): + self.raise_error(f"expected {type}/{value}, got {self.type}/{self.value}") + value = self.value + self.gettoken() + return value + + def gettoken(self) -> None: + tup = next(self.generator) + while tup[0] in (tokenize.COMMENT, tokenize.NL): + tup = next(self.generator) + self.type, self.value, self.begin, self.end, self.line = tup + # print token.tok_name[self.type], repr(self.value) + + def raise_error(self, msg: str) -> NoReturn: + raise SyntaxError( + msg, (str(self.filename), self.end[0], self.end[1], self.line) + ) + + +class NFAState: + arcs: list[tuple[str | None, "NFAState"]] + + def __init__(self) -> None: + self.arcs = [] # list of (label, NFAState) pairs + + def addarc(self, next: "NFAState", label: str | None = None) -> None: + assert label is None or isinstance(label, str) + assert isinstance(next, NFAState) + self.arcs.append((label, next)) + + +class DFAState: + nfaset: dict[NFAState, Any] + isfinal: bool + arcs: dict[str, "DFAState"] + + def __init__(self, nfaset: dict[NFAState, Any], final: NFAState) -> None: + assert isinstance(nfaset, dict) + assert isinstance(next(iter(nfaset)), NFAState) + assert isinstance(final, NFAState) + self.nfaset = nfaset + self.isfinal = final in nfaset + self.arcs = {} # map from label to DFAState + + def addarc(self, next: "DFAState", label: str) -> None: + assert isinstance(label, str) + assert label not in self.arcs + assert isinstance(next, DFAState) + self.arcs[label] = next + + def unifystate(self, old: "DFAState", new: "DFAState") -> None: + for label, next in self.arcs.items(): + if next is old: + self.arcs[label] = new + + def __eq__(self, other: Any) -> bool: + # Equality test -- ignore the nfaset instance variable + assert isinstance(other, DFAState) + if self.isfinal != other.isfinal: + return False + # Can't just return self.arcs == other.arcs, because that + # would invoke this method recursively, with cycles... + if len(self.arcs) != len(other.arcs): + return False + for label, next in self.arcs.items(): + if next is not other.arcs.get(label): + return False + return True + + __hash__: Any = None # For Py3 compatibility. + + +def generate_grammar(filename: Path = "Grammar.txt") -> PgenGrammar: + p = ParserGenerator(filename) + return p.make_grammar() diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b8c6d5d8136eefaf8499856edf8599254256b48f Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.py new file mode 100644 index 0000000000000000000000000000000000000000..8b531ee5c64cddc8de543c80d275647591d5dd1f --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/token.py @@ -0,0 +1,95 @@ +"""Token constants (from "token.h").""" + +from typing import Final + +# Taken from Python (r53757) and modified to include some tokens +# originally monkeypatched in by pgen2.tokenize + +# --start constants-- +ENDMARKER: Final = 0 +NAME: Final = 1 +NUMBER: Final = 2 +STRING: Final = 3 +NEWLINE: Final = 4 +INDENT: Final = 5 +DEDENT: Final = 6 +LPAR: Final = 7 +RPAR: Final = 8 +LSQB: Final = 9 +RSQB: Final = 10 +COLON: Final = 11 +COMMA: Final = 12 +SEMI: Final = 13 +PLUS: Final = 14 +MINUS: Final = 15 +STAR: Final = 16 +SLASH: Final = 17 +VBAR: Final = 18 +AMPER: Final = 19 +LESS: Final = 20 +GREATER: Final = 21 +EQUAL: Final = 22 +DOT: Final = 23 +PERCENT: Final = 24 +BACKQUOTE: Final = 25 +LBRACE: Final = 26 +RBRACE: Final = 27 +EQEQUAL: Final = 28 +NOTEQUAL: Final = 29 +LESSEQUAL: Final = 30 +GREATEREQUAL: Final = 31 +TILDE: Final = 32 +CIRCUMFLEX: Final = 33 +LEFTSHIFT: Final = 34 +RIGHTSHIFT: Final = 35 +DOUBLESTAR: Final = 36 +PLUSEQUAL: Final = 37 +MINEQUAL: Final = 38 +STAREQUAL: Final = 39 +SLASHEQUAL: Final = 40 +PERCENTEQUAL: Final = 41 +AMPEREQUAL: Final = 42 +VBAREQUAL: Final = 43 +CIRCUMFLEXEQUAL: Final = 44 +LEFTSHIFTEQUAL: Final = 45 +RIGHTSHIFTEQUAL: Final = 46 +DOUBLESTAREQUAL: Final = 47 +DOUBLESLASH: Final = 48 +DOUBLESLASHEQUAL: Final = 49 +AT: Final = 50 +ATEQUAL: Final = 51 +OP: Final = 52 +COMMENT: Final = 53 +NL: Final = 54 +RARROW: Final = 55 +AWAIT: Final = 56 +ASYNC: Final = 57 +ERRORTOKEN: Final = 58 +COLONEQUAL: Final = 59 +FSTRING_START: Final = 60 +FSTRING_MIDDLE: Final = 61 +FSTRING_END: Final = 62 +BANG: Final = 63 +TSTRING_START: Final = 64 +TSTRING_MIDDLE: Final = 65 +TSTRING_END: Final = 66 +N_TOKENS: Final = 67 +NT_OFFSET: Final = 256 +# --end constants-- + +tok_name: Final[dict[int, str]] = {} +for _name, _value in list(globals().items()): + if type(_value) is int: + tok_name[_value] = _name + + +def ISTERMINAL(x: int) -> bool: + return x < NT_OFFSET + + +def ISNONTERMINAL(x: int) -> bool: + return x >= NT_OFFSET + + +def ISEOF(x: int) -> bool: + return x == ENDMARKER diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.cpython-311-x86_64-linux-gnu.so b/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.cpython-311-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..f24cb9004c0aec40ce80d7c25f635d09157f409d Binary files /dev/null and b/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.cpython-311-x86_64-linux-gnu.so differ diff --git a/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.py b/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.py new file mode 100644 index 0000000000000000000000000000000000000000..4e3761f3028a0324352ade4385f9053387304147 --- /dev/null +++ b/py311/lib/python3.11/site-packages/blib2to3/pgen2/tokenize.py @@ -0,0 +1,226 @@ +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation. +# All rights reserved. + +# mypy: allow-untyped-defs, allow-untyped-calls + +"""Tokenization help for Python programs. + +generate_tokens(readline) is a generator that breaks a stream of +text into Python tokens. It accepts a readline-like method which is called +repeatedly to get the next line of input (or "" for EOF). It generates +5-tuples with these members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators + +Older entry points + tokenize_loop(readline, tokeneater) + tokenize(readline, tokeneater=printtoken) +are the same, except instead of generating tokens, tokeneater is a callback +function to which the 5 fields described above are passed as 5 arguments, +each time a new token is found.""" + +import sys +from collections.abc import Iterator + +from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.token import ( + ASYNC, + AWAIT, + COMMENT, + DEDENT, + ENDMARKER, + FSTRING_END, + FSTRING_MIDDLE, + FSTRING_START, + INDENT, + NAME, + NEWLINE, + NL, + NUMBER, + OP, + STRING, + TSTRING_END, + TSTRING_MIDDLE, + TSTRING_START, + tok_name, +) + +__author__ = "Ka-Ping Yee " +__credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro" + +import pytokens +from pytokens import TokenType + +from . import token as _token + +__all__ = [x for x in dir(_token) if x[0] != "_"] + [ + "tokenize", + "generate_tokens", + "untokenize", +] +del _token + +Coord = tuple[int, int] +TokenInfo = tuple[int, str, Coord, Coord, str] + +TOKEN_TYPE_MAP = { + TokenType.indent: INDENT, + TokenType.dedent: DEDENT, + TokenType.newline: NEWLINE, + TokenType.nl: NL, + TokenType.comment: COMMENT, + TokenType.semicolon: OP, + TokenType.lparen: OP, + TokenType.rparen: OP, + TokenType.lbracket: OP, + TokenType.rbracket: OP, + TokenType.lbrace: OP, + TokenType.rbrace: OP, + TokenType.colon: OP, + TokenType.op: OP, + TokenType.identifier: NAME, + TokenType.number: NUMBER, + TokenType.string: STRING, + TokenType.fstring_start: FSTRING_START, + TokenType.fstring_middle: FSTRING_MIDDLE, + TokenType.fstring_end: FSTRING_END, + TokenType.tstring_start: TSTRING_START, + TokenType.tstring_middle: TSTRING_MIDDLE, + TokenType.tstring_end: TSTRING_END, + TokenType.endmarker: ENDMARKER, +} + + +class TokenError(Exception): ... + + +def transform_whitespace( + token: pytokens.Token, source: str, prev_token: pytokens.Token | None +) -> pytokens.Token: + r""" + Black treats `\\\n` at the end of a line as a 'NL' token, while it + is ignored as whitespace in the regular Python parser. + But, only the first one. If there's a `\\\n` following it + (as in, a \ just by itself on a line), that is not made into NL. + """ + if ( + token.type == TokenType.whitespace + and prev_token is not None + and prev_token.type not in (TokenType.nl, TokenType.newline) + ): + token_str = source[token.start_index : token.end_index] + if token_str.startswith("\\\r\n"): + return pytokens.Token( + TokenType.nl, + token.start_index, + token.start_index + 3, + token.start_line, + token.start_col, + token.start_line, + token.start_col + 3, + ) + elif token_str.startswith("\\\n") or token_str.startswith("\\\r"): + return pytokens.Token( + TokenType.nl, + token.start_index, + token.start_index + 2, + token.start_line, + token.start_col, + token.start_line, + token.start_col + 2, + ) + + return token + + +def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]: + lines = source.split("\n") + lines += [""] # For newline tokens in files that don't end in a newline + line, column = 1, 0 + + prev_token: pytokens.Token | None = None + try: + for token in pytokens.tokenize(source): + token = transform_whitespace(token, source, prev_token) + + line, column = token.start_line, token.start_col + if token.type == TokenType.whitespace: + continue + + token_str = source[token.start_index : token.end_index] + + if token.type == TokenType.newline and token_str == "": + # Black doesn't yield empty newline tokens at the end of a file + # if there's no newline at the end of a file. + prev_token = token + continue + + source_line = lines[token.start_line - 1] + + if token.type == TokenType.identifier and token_str in ("async", "await"): + # Black uses `async` and `await` token types just for those two keywords + yield ( + ASYNC if token_str == "async" else AWAIT, + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, + ) + elif token.type == TokenType.op and token_str == "...": + # Black doesn't have an ellipsis token yet, yield 3 DOTs instead + assert token.start_line == token.end_line + assert token.end_col == token.start_col + 3 + + token_str = "." + for start_col in range(token.start_col, token.start_col + 3): + end_col = start_col + 1 + yield ( + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, start_col), + (token.end_line, end_col), + source_line, + ) + else: + token_type = TOKEN_TYPE_MAP.get(token.type) + if token_type is None: + raise ValueError(f"Unknown token type: {token.type!r}") + yield ( + TOKEN_TYPE_MAP[token.type], + token_str, + (token.start_line, token.start_col), + (token.end_line, token.end_col), + source_line, + ) + prev_token = token + + except pytokens.UnexpectedEOF: + raise TokenError("Unexpected EOF in multi-line statement", (line, column)) + except pytokens.TokenizeError as exc: + raise TokenError(f"Failed to parse: {type(exc).__name__}", (line, column)) + + +def printtoken( + type: int, token: str, srow_col: Coord, erow_col: Coord, line: str +) -> None: # for testing + srow, scol = srow_col + erow, ecol = erow_col + print(f"{srow},{scol}-{erow},{ecol}:\t{tok_name[type]}\t{token!r}") + + +if __name__ == "__main__": # testing + if len(sys.argv) > 1: + token_iterator = tokenize(open(sys.argv[1]).read()) + else: + token_iterator = tokenize(sys.stdin.read()) + + for tok in token_iterator: + printtoken(*tok) diff --git a/py311/lib/python3.11/site-packages/google/api_core/__init__.py b/py311/lib/python3.11/site-packages/google/api_core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a52ffe8741d24a477cba5781884d8c14dd987a32 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/__init__.py @@ -0,0 +1,41 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Google API Core. + +This package contains common code and utilities used by Google client libraries. +""" + +from google.api_core import _python_package_support +from google.api_core import _python_version_support +from google.api_core import version as api_core_version + +__version__ = api_core_version.__version__ + +# NOTE: Until dependent artifacts require this version of +# google.api_core, the functionality below must be made available +# manually in those artifacts. + +# expose dependency checks for external callers +check_python_version = _python_version_support.check_python_version +check_dependency_versions = _python_package_support.check_dependency_versions +parse_version_to_tuple = _python_package_support.parse_version_to_tuple +warn_deprecation_for_versions_less_than = ( + _python_package_support.warn_deprecation_for_versions_less_than +) +DependencyConstraint = _python_package_support.DependencyConstraint + +# perform version checks against api_core, and emit warnings if needed +check_python_version(package="google.api_core") +check_dependency_versions("google.api_core") diff --git a/py311/lib/python3.11/site-packages/google/api_core/_python_package_support.py b/py311/lib/python3.11/site-packages/google/api_core/_python_package_support.py new file mode 100644 index 0000000000000000000000000000000000000000..06da2bb00e62c7360725a82ec47b27334f0d3b2c --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/_python_package_support.py @@ -0,0 +1,234 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code to check versions of dependencies used by Google Cloud Client Libraries.""" + +import warnings +import sys +from typing import Optional, Tuple + +from collections import namedtuple + +from ._python_version_support import ( + _flatten_message, + _get_distribution_and_import_packages, +) + +if sys.version_info >= (3, 8): + from importlib import metadata +else: + # TODO(https://github.com/googleapis/python-api-core/issues/835): Remove + # this code path once we drop support for Python 3.7 + import importlib_metadata as metadata + +ParsedVersion = Tuple[int, ...] + +# Here we list all the packages for which we want to issue warnings +# about deprecated and unsupported versions. +DependencyConstraint = namedtuple( + "DependencyConstraint", + ["package_name", "minimum_fully_supported_version", "recommended_version"], +) +_PACKAGE_DEPENDENCY_WARNINGS = [ + DependencyConstraint( + "google.protobuf", + minimum_fully_supported_version="4.25.8", + recommended_version="6.x", + ) +] + + +DependencyVersion = namedtuple("DependencyVersion", ["version", "version_string"]) +# Version string we provide in a DependencyVersion when we can't determine the version of a +# package. +UNKNOWN_VERSION_STRING = "--" + + +def parse_version_to_tuple(version_string: str) -> ParsedVersion: + """Safely converts a semantic version string to a comparable tuple of integers. + + Example: "4.25.8" -> (4, 25, 8) + Ignores non-numeric parts and handles common version formats. + + Args: + version_string: Version string in the format "x.y.z" or "x.y.z" + + Returns: + Tuple of integers for the parsed version string. + """ + parts = [] + for part in version_string.split("."): + try: + parts.append(int(part)) + except ValueError: + # If it's a non-numeric part (e.g., '1.0.0b1' -> 'b1'), stop here. + # This is a simplification compared to 'packaging.parse_version', but sufficient + # for comparing strictly numeric semantic versions. + break + return tuple(parts) + + +def get_dependency_version( + dependency_name: str, +) -> DependencyVersion: + """Get the parsed version of an installed package dependency. + + This function checks for an installed package and returns its version + as a comparable tuple of integers object for safe comparison. It handles + both modern (Python 3.8+) and legacy (Python 3.7) environments. + + Args: + dependency_name: The distribution name of the package (e.g., 'requests'). + + Returns: + A DependencyVersion namedtuple with `version` (a tuple of integers) and + `version_string` attributes, or `DependencyVersion(None, + UNKNOWN_VERSION_STRING)` if the package is not found or + another error occurs during version discovery. + + """ + try: + version_string: str = metadata.version(dependency_name) + parsed_version = parse_version_to_tuple(version_string) + return DependencyVersion(parsed_version, version_string) + except Exception: + # Catch exceptions from metadata.version() (e.g., PackageNotFoundError) + # or errors during parse_version_to_tuple + return DependencyVersion(None, UNKNOWN_VERSION_STRING) + + +def warn_deprecation_for_versions_less_than( + consumer_import_package: str, + dependency_import_package: str, + minimum_fully_supported_version: str, + recommended_version: Optional[str] = None, + message_template: Optional[str] = None, +): + """Issue any needed deprecation warnings for `dependency_import_package`. + + If `dependency_import_package` is installed at a version less than + `minimum_fully_supported_version`, this issues a warning using either a + default `message_template` or one provided by the user. The + default `message_template` informs the user that they will not receive + future updates for `consumer_import_package` if + `dependency_import_package` is somehow pinned to a version lower + than `minimum_fully_supported_version`. + + Args: + consumer_import_package: The import name of the package that + needs `dependency_import_package`. + dependency_import_package: The import name of the dependency to check. + minimum_fully_supported_version: The dependency_import_package version number + below which a deprecation warning will be logged. + recommended_version: If provided, the recommended next version, which + could be higher than `minimum_fully_supported_version`. + message_template: A custom default message template to replace + the default. This `message_template` is treated as an + f-string, where the following variables are defined: + `dependency_import_package`, `consumer_import_package` and + `dependency_distribution_package` and + `consumer_distribution_package` and `dependency_package`, + `consumer_package` , which contain the import packages, the + distribution packages, and pretty string with both the + distribution and import packages for the dependency and the + consumer, respectively; and `minimum_fully_supported_version`, + `version_used`, and `version_used_string`, which refer to supported + and currently-used versions of the dependency. + + """ + if ( + not consumer_import_package + or not dependency_import_package + or not minimum_fully_supported_version + ): # pragma: NO COVER + return + + dependency_version = get_dependency_version(dependency_import_package) + if not dependency_version.version: + return + + if dependency_version.version < parse_version_to_tuple( + minimum_fully_supported_version + ): + ( + dependency_package, + dependency_distribution_package, + ) = _get_distribution_and_import_packages(dependency_import_package) + ( + consumer_package, + consumer_distribution_package, + ) = _get_distribution_and_import_packages(consumer_import_package) + + recommendation = ( + " (we recommend {recommended_version})" if recommended_version else "" + ) + message_template = message_template or _flatten_message( + """ + DEPRECATION: Package {consumer_package} depends on + {dependency_package}, currently installed at version + {version_used_string}. Future updates to + {consumer_package} will require {dependency_package} at + version {minimum_fully_supported_version} or + higher{recommendation}. Please ensure that either (a) your + Python environment doesn't pin the version of + {dependency_package}, so that updates to + {consumer_package} can require the higher version, or (b) + you manually update your Python environment to use at + least version {minimum_fully_supported_version} of + {dependency_package}. + """ + ) + warnings.warn( + message_template.format( + consumer_import_package=consumer_import_package, + dependency_import_package=dependency_import_package, + consumer_distribution_package=consumer_distribution_package, + dependency_distribution_package=dependency_distribution_package, + dependency_package=dependency_package, + consumer_package=consumer_package, + minimum_fully_supported_version=minimum_fully_supported_version, + recommendation=recommendation, + version_used=dependency_version.version, + version_used_string=dependency_version.version_string, + ), + FutureWarning, + ) + + +def check_dependency_versions( + consumer_import_package: str, *package_dependency_warnings: DependencyConstraint +): + """Bundle checks for all package dependencies. + + This function can be called by all consumers of google.api_core, + to emit needed deprecation warnings for any of their + dependencies. The dependencies to check can be passed as arguments, or if + none are provided, it will default to the list in + `_PACKAGE_DEPENDENCY_WARNINGS`. + + Args: + consumer_import_package: The distribution name of the calling package, whose + dependencies we're checking. + *package_dependency_warnings: A variable number of DependencyConstraint + objects, each specifying a dependency to check. + """ + if not package_dependency_warnings: + package_dependency_warnings = tuple(_PACKAGE_DEPENDENCY_WARNINGS) + for package_info in package_dependency_warnings: + warn_deprecation_for_versions_less_than( + consumer_import_package, + package_info.package_name, + package_info.minimum_fully_supported_version, + recommended_version=package_info.recommended_version, + ) diff --git a/py311/lib/python3.11/site-packages/google/api_core/_python_version_support.py b/py311/lib/python3.11/site-packages/google/api_core/_python_version_support.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c0dfe1bcfd55d263e5eef47986194b431b34d4 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/_python_version_support.py @@ -0,0 +1,278 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code to check Python versions supported by Google Cloud Client Libraries.""" + +import datetime +import enum +import logging +import warnings +import sys +import textwrap +from typing import Any, List, NamedTuple, Optional, Dict, Tuple + + +_LOGGER = logging.getLogger(__name__) + + +class PythonVersionStatus(enum.Enum): + """Support status of a Python version in this client library artifact release. + + "Support", in this context, means that this release of a client library + artifact is configured to run on the currently configured version of + Python. + """ + + PYTHON_VERSION_STATUS_UNSPECIFIED = "PYTHON_VERSION_STATUS_UNSPECIFIED" + + PYTHON_VERSION_SUPPORTED = "PYTHON_VERSION_SUPPORTED" + """This Python version is fully supported, so the artifact running on this + version will have all features and bug fixes.""" + + PYTHON_VERSION_DEPRECATED = "PYTHON_VERSION_DEPRECATED" + """This Python version is still supported, but support will end within a + year. At that time, there will be no more releases for this artifact + running under this Python version.""" + + PYTHON_VERSION_EOL = "PYTHON_VERSION_EOL" + """This Python version has reached its end of life in the Python community + (see https://devguide.python.org/versions/), and this artifact will cease + supporting this Python version within the next few releases.""" + + PYTHON_VERSION_UNSUPPORTED = "PYTHON_VERSION_UNSUPPORTED" + """This release of the client library artifact may not be the latest, since + current releases no longer support this Python version.""" + + +class VersionInfo(NamedTuple): + """Hold release and support date information for a Python version.""" + + version: str + python_beta: Optional[datetime.date] + python_start: datetime.date + python_eol: datetime.date + gapic_start: Optional[datetime.date] = None # unused + gapic_deprecation: Optional[datetime.date] = None + gapic_end: Optional[datetime.date] = None + dep_unpatchable_cve: Optional[datetime.date] = None # unused + + +PYTHON_VERSIONS: List[VersionInfo] = [ + # Refer to https://devguide.python.org/versions/ and the PEPs linked therefrom. + VersionInfo( + version="3.7", + python_beta=None, + python_start=datetime.date(2018, 6, 27), + python_eol=datetime.date(2023, 6, 27), + ), + VersionInfo( + version="3.8", + python_beta=None, + python_start=datetime.date(2019, 10, 14), + python_eol=datetime.date(2024, 10, 7), + ), + VersionInfo( + version="3.9", + python_beta=datetime.date(2020, 5, 18), + python_start=datetime.date(2020, 10, 5), + python_eol=datetime.date(2025, 10, 5), + gapic_end=datetime.date(2025, 10, 5) + datetime.timedelta(days=90), + ), + VersionInfo( + version="3.10", + python_beta=datetime.date(2021, 5, 3), + python_start=datetime.date(2021, 10, 4), + python_eol=datetime.date(2026, 10, 4), # TODO: specify day when announced + ), + VersionInfo( + version="3.11", + python_beta=datetime.date(2022, 5, 8), + python_start=datetime.date(2022, 10, 24), + python_eol=datetime.date(2027, 10, 24), # TODO: specify day when announced + ), + VersionInfo( + version="3.12", + python_beta=datetime.date(2023, 5, 22), + python_start=datetime.date(2023, 10, 2), + python_eol=datetime.date(2028, 10, 2), # TODO: specify day when announced + ), + VersionInfo( + version="3.13", + python_beta=datetime.date(2024, 5, 8), + python_start=datetime.date(2024, 10, 7), + python_eol=datetime.date(2029, 10, 7), # TODO: specify day when announced + ), + VersionInfo( + version="3.14", + python_beta=datetime.date(2025, 5, 7), + python_start=datetime.date(2025, 10, 7), + python_eol=datetime.date(2030, 10, 7), # TODO: specify day when announced + ), +] + +PYTHON_VERSION_INFO: Dict[Tuple[int, int], VersionInfo] = {} +for info in PYTHON_VERSIONS: + major, minor = map(int, info.version.split(".")) + PYTHON_VERSION_INFO[(major, minor)] = info + + +LOWEST_TRACKED_VERSION = min(PYTHON_VERSION_INFO.keys()) +_FAKE_PAST_DATE = datetime.date.min + datetime.timedelta(days=900) +_FAKE_PAST_VERSION = VersionInfo( + version="0.0", + python_beta=_FAKE_PAST_DATE, + python_start=_FAKE_PAST_DATE, + python_eol=_FAKE_PAST_DATE, +) +_FAKE_FUTURE_DATE = datetime.date.max - datetime.timedelta(days=900) +_FAKE_FUTURE_VERSION = VersionInfo( + version="999.0", + python_beta=_FAKE_FUTURE_DATE, + python_start=_FAKE_FUTURE_DATE, + python_eol=_FAKE_FUTURE_DATE, +) +DEPRECATION_WARNING_PERIOD = datetime.timedelta(days=365) +EOL_GRACE_PERIOD = datetime.timedelta(weeks=1) + + +def _flatten_message(text: str) -> str: + """Dedent a multi-line string and flatten it into a single line.""" + return " ".join(textwrap.dedent(text).strip().split()) + + +# TODO(https://github.com/googleapis/python-api-core/issues/835): +# Remove once we no longer support Python 3.9. +# `importlib.metadata.packages_distributions()` is only supported in Python 3.10 and newer +# https://docs.python.org/3/library/importlib.metadata.html#importlib.metadata.packages_distributions +if sys.version_info < (3, 10): + + def _get_pypi_package_name(module_name): # pragma: NO COVER + """Determine the PyPI package name for a given module name.""" + return None + +else: + from importlib import metadata + + def _get_pypi_package_name(module_name): + """Determine the PyPI package name for a given module name.""" + try: + # Get the mapping of modules to distributions + module_to_distributions = metadata.packages_distributions() + + # Check if the module is found in the mapping + if module_name in module_to_distributions: # pragma: NO COVER + # The value is a list of distribution names, take the first one + return module_to_distributions[module_name][0] + except Exception as e: # pragma: NO COVER + _LOGGER.info( + "An error occurred while determining PyPI package name for %s: %s", + module_name, + e, + ) + + return None + + +def _get_distribution_and_import_packages(import_package: str) -> Tuple[str, Any]: + """Return a pretty string with distribution & import package names.""" + distribution_package = _get_pypi_package_name(import_package) + dependency_distribution_and_import_packages = ( + f"package {distribution_package} ({import_package})" + if distribution_package + else import_package + ) + return dependency_distribution_and_import_packages, distribution_package + + +def check_python_version( + package: str = "this package", today: Optional[datetime.date] = None +) -> PythonVersionStatus: + """Check the running Python version and issue a support warning if needed. + + Args: + today: The date to check against. Defaults to the current date. + + Returns: + The support status of the current Python version. + """ + today = today or datetime.date.today() + package_label, _ = _get_distribution_and_import_packages(package) + + python_version = sys.version_info + version_tuple = (python_version.major, python_version.minor) + py_version_str = sys.version.split()[0] + + version_info = PYTHON_VERSION_INFO.get(version_tuple) + + if not version_info: + if version_tuple < LOWEST_TRACKED_VERSION: + version_info = _FAKE_PAST_VERSION + else: + version_info = _FAKE_FUTURE_VERSION + + gapic_deprecation = version_info.gapic_deprecation or ( + version_info.python_eol - DEPRECATION_WARNING_PERIOD + ) + gapic_end = version_info.gapic_end or (version_info.python_eol + EOL_GRACE_PERIOD) + + def min_python(date: datetime.date) -> str: + """Find the minimum supported Python version for a given date.""" + for version, info in sorted(PYTHON_VERSION_INFO.items()): + if info.python_start <= date < info.python_eol: + return f"{version[0]}.{version[1]}" + return "at a currently supported version [https://devguide.python.org/versions]" + + if gapic_end < today: + message = _flatten_message( + f""" + You are using a non-supported Python version ({py_version_str}). + Google will not post any further updates to {package_label} + supporting this Python version. Please upgrade to the latest Python + version, or at least Python {min_python(today)}, and then update + {package_label}. + """ + ) + warnings.warn(message, FutureWarning) + return PythonVersionStatus.PYTHON_VERSION_UNSUPPORTED + + eol_date = version_info.python_eol + EOL_GRACE_PERIOD + if eol_date <= today <= gapic_end: + message = _flatten_message( + f""" + You are using a Python version ({py_version_str}) + past its end of life. Google will update {package_label} + with critical bug fixes on a best-effort basis, but not + with any other fixes or features. Please upgrade + to the latest Python version, or at least Python + {min_python(today)}, and then update {package_label}. + """ + ) + warnings.warn(message, FutureWarning) + return PythonVersionStatus.PYTHON_VERSION_EOL + + if gapic_deprecation <= today <= gapic_end: + message = _flatten_message( + f""" + You are using a Python version ({py_version_str}) which Google will + stop supporting in new releases of {package_label} once it reaches + its end of life ({version_info.python_eol}). Please upgrade to the + latest Python version, or at least Python + {min_python(version_info.python_eol)}, to continue receiving updates + for {package_label} past that date. + """ + ) + warnings.warn(message, FutureWarning) + return PythonVersionStatus.PYTHON_VERSION_DEPRECATED + + return PythonVersionStatus.PYTHON_VERSION_SUPPORTED diff --git a/py311/lib/python3.11/site-packages/google/api_core/_rest_streaming_base.py b/py311/lib/python3.11/site-packages/google/api_core/_rest_streaming_base.py new file mode 100644 index 0000000000000000000000000000000000000000..3bc87a963e108befb5b21618b2c935881b0e776b --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/_rest_streaming_base.py @@ -0,0 +1,118 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for server-side streaming in REST.""" + +from collections import deque +import string +from typing import Deque, Union +import types + +import proto +import google.protobuf.message +from google.protobuf.json_format import Parse + + +class BaseResponseIterator: + """Base Iterator over REST API responses. This class should not be used directly. + + Args: + response_message_cls (Union[proto.Message, google.protobuf.message.Message]): A response + class expected to be returned from an API. + + Raises: + ValueError: If `response_message_cls` is not a subclass of `proto.Message` or `google.protobuf.message.Message`. + """ + + def __init__( + self, + response_message_cls: Union[proto.Message, google.protobuf.message.Message], + ): + self._response_message_cls = response_message_cls + # Contains a list of JSON responses ready to be sent to user. + self._ready_objs: Deque[str] = deque() + # Current JSON response being built. + self._obj = "" + # Keeps track of the nesting level within a JSON object. + self._level = 0 + # Keeps track whether HTTP response is currently sending values + # inside of a string value. + self._in_string = False + # Whether an escape symbol "\" was encountered. + self._escape_next = False + + self._grab = types.MethodType(self._create_grab(), self) + + def _process_chunk(self, chunk: str): + if self._level == 0: + if chunk[0] != "[": + raise ValueError( + "Can only parse array of JSON objects, instead got %s" % chunk + ) + for char in chunk: + if char == "{": + if self._level == 1: + # Level 1 corresponds to the outermost JSON object + # (i.e. the one we care about). + self._obj = "" + if not self._in_string: + self._level += 1 + self._obj += char + elif char == "}": + self._obj += char + if not self._in_string: + self._level -= 1 + if not self._in_string and self._level == 1: + self._ready_objs.append(self._obj) + elif char == '"': + # Helps to deal with an escaped quotes inside of a string. + if not self._escape_next: + self._in_string = not self._in_string + self._obj += char + elif char in string.whitespace: + if self._in_string: + self._obj += char + elif char == "[": + if self._level == 0: + self._level += 1 + else: + self._obj += char + elif char == "]": + if self._level == 1: + self._level -= 1 + else: + self._obj += char + else: + self._obj += char + self._escape_next = not self._escape_next if char == "\\" else False + + def _create_grab(self): + if issubclass(self._response_message_cls, proto.Message): + + def grab(this): + return this._response_message_cls.from_json( + this._ready_objs.popleft(), ignore_unknown_fields=True + ) + + return grab + elif issubclass(self._response_message_cls, google.protobuf.message.Message): + + def grab(this): + return Parse(this._ready_objs.popleft(), this._response_message_cls()) + + return grab + else: + raise ValueError( + "Response message class must be a subclass of proto.Message or google.protobuf.message.Message." + ) diff --git a/py311/lib/python3.11/site-packages/google/api_core/bidi.py b/py311/lib/python3.11/site-packages/google/api_core/bidi.py new file mode 100644 index 0000000000000000000000000000000000000000..7f45c2af185b6a3f57b75a89badd1fc17682d52a --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/bidi.py @@ -0,0 +1,735 @@ +# Copyright 2017, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for synchronous bidirectional streaming RPCs.""" + +import collections +import datetime +import logging +import queue as queue_module +import threading +import time + +from google.api_core import exceptions +from google.api_core.bidi_base import BidiRpcBase + +_LOGGER = logging.getLogger(__name__) +_BIDIRECTIONAL_CONSUMER_NAME = "Thread-ConsumeBidirectionalStream" + + +class _RequestQueueGenerator(object): + """A helper for sending requests to a gRPC stream from a Queue. + + This generator takes requests off a given queue and yields them to gRPC. + + This helper is useful when you have an indeterminate, indefinite, or + otherwise open-ended set of requests to send through a request-streaming + (or bidirectional) RPC. + + + Example:: + + requests = request_queue_generator(q) + call = stub.StreamingRequest(iter(requests)) + requests.call = call + + for response in call: + print(response) + q.put(...) + + + Args: + queue (queue_module.Queue): The request queue. + period (float): The number of seconds to wait for items from the queue + before checking if the RPC is cancelled. In practice, this + determines the maximum amount of time the request consumption + thread will live after the RPC is cancelled. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is done independently of the request queue to allow fo + easily restarting streams that require some initial configuration + request. + """ + + def __init__(self, queue, period=1, initial_request=None): + self._queue = queue + self._period = period + self._initial_request = initial_request + self.call = None + + def _is_active(self): + # Note: there is a possibility that this starts *before* the call + # property is set. So we have to check if self.call is set before + # seeing if it's active. We need to return True if self.call is None. + # See https://github.com/googleapis/python-api-core/issues/560. + return self.call is None or self.call.is_active() + + def __iter__(self): + # The reason this is necessary is because gRPC takes an iterator as the + # request for request-streaming RPCs. gRPC consumes this iterator in + # another thread to allow it to block while generating requests for + # the stream. However, if the generator blocks indefinitely gRPC will + # not be able to clean up the thread as it'll be blocked on + # `next(iterator)` and not be able to check the channel status to stop + # iterating. This helper mitigates that by waiting on the queue with + # a timeout and checking the RPC state before yielding. + # + # Finally, it allows for retrying without swapping queues because if + # it does pull an item off the queue when the RPC is inactive, it'll + # immediately put it back and then exit. This is necessary because + # yielding the item in this case will cause gRPC to discard it. In + # practice, this means that the order of messages is not guaranteed. + # If such a thing is necessary it would be easy to use a priority + # queue. + # + # Note that it is possible to accomplish this behavior without + # "spinning" (using a queue timeout). One possible way would be to use + # more threads to multiplex the grpc end event with the queue, another + # possible way is to use selectors and a custom event/queue object. + # Both of these approaches are significant from an engineering + # perspective for small benefit - the CPU consumed by spinning is + # pretty minuscule. + + if self._initial_request is not None: + if callable(self._initial_request): + yield self._initial_request() + else: + yield self._initial_request + + while True: + try: + item = self._queue.get(timeout=self._period) + except queue_module.Empty: + if not self._is_active(): + _LOGGER.debug( + "Empty queue and inactive call, exiting request " "generator." + ) + return + else: + # call is still active, keep waiting for queue items. + continue + + # The consumer explicitly sent "None", indicating that the request + # should end. + if item is None: + _LOGGER.debug("Cleanly exiting request generator.") + return + + if not self._is_active(): + # We have an item, but the call is closed. We should put the + # item back on the queue so that the next call can consume it. + self._queue.put(item) + _LOGGER.debug( + "Inactive call, replacing item on queue and exiting " + "request generator." + ) + return + + yield item + + +class _Throttle(object): + """A context manager limiting the total entries in a sliding time window. + + If more than ``access_limit`` attempts are made to enter the context manager + instance in the last ``time window`` interval, the exceeding requests block + until enough time elapses. + + The context manager instances are thread-safe and can be shared between + multiple threads. If multiple requests are blocked and waiting to enter, + the exact order in which they are allowed to proceed is not determined. + + Example:: + + max_three_per_second = _Throttle( + access_limit=3, time_window=datetime.timedelta(seconds=1) + ) + + for i in range(5): + with max_three_per_second as time_waited: + print("{}: Waited {} seconds to enter".format(i, time_waited)) + + Args: + access_limit (int): the maximum number of entries allowed in the time window + time_window (datetime.timedelta): the width of the sliding time window + """ + + def __init__(self, access_limit, time_window): + if access_limit < 1: + raise ValueError("access_limit argument must be positive") + + if time_window <= datetime.timedelta(0): + raise ValueError("time_window argument must be a positive timedelta") + + self._time_window = time_window + self._access_limit = access_limit + self._past_entries = collections.deque( + maxlen=access_limit + ) # least recent first + self._entry_lock = threading.Lock() + + def __enter__(self): + with self._entry_lock: + cutoff_time = datetime.datetime.now() - self._time_window + + # drop the entries that are too old, as they are no longer relevant + while self._past_entries and self._past_entries[0] < cutoff_time: + self._past_entries.popleft() + + if len(self._past_entries) < self._access_limit: + self._past_entries.append(datetime.datetime.now()) + return 0.0 # no waiting was needed + + to_wait = (self._past_entries[0] - cutoff_time).total_seconds() + time.sleep(to_wait) + + self._past_entries.append(datetime.datetime.now()) + return to_wait + + def __exit__(self, *_): + pass + + def __repr__(self): + return "{}(access_limit={}, time_window={})".format( + self.__class__.__name__, self._access_limit, repr(self._time_window) + ) + + +class BidiRpc(BidiRpcBase): + """A helper for consuming a bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + Example:: + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + rpc = BidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + metadata=[('name', 'value')] + ) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + rpc.close() + + This does *not* retry the stream on errors. See :class:`ResumableBidiRpc`. + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + metadata (Sequence[Tuple(str, str)]): RPC metadata to include in + the request. + """ + + def _create_queue(self): + """Create a queue for requests.""" + return queue_module.Queue() + + def open(self): + """Opens the stream.""" + if self.is_active: + raise ValueError("Cannot open an already open stream.") + + request_generator = _RequestQueueGenerator( + self._request_queue, initial_request=self._initial_request + ) + try: + call = self._start_rpc(iter(request_generator), metadata=self._rpc_metadata) + except exceptions.GoogleAPICallError as exc: + # The original `grpc.RpcError` (which is usually also a `grpc.Call`) is + # available from the ``response`` property on the mapped exception. + self._on_call_done(exc.response) + raise + + request_generator.call = call + + # TODO: api_core should expose the future interface for wrapped + # callables as well. + if hasattr(call, "_wrapped"): # pragma: NO COVER + call._wrapped.add_done_callback(self._on_call_done) + else: + call.add_done_callback(self._on_call_done) + + self._request_generator = request_generator + self.call = call + + def close(self): + """Closes the stream.""" + if self.call is not None: + self.call.cancel() + + # Put None in request queue to signal termination. + self._request_queue.put(None) + self._request_generator = None + self._initial_request = None + self._callbacks = [] + # Don't set self.call to None. Keep it around so that send/recv can + # raise the error. + + def send(self, request): + """Queue a message to be sent on the stream. + + Send is non-blocking. + + If the underlying RPC has been closed, this will raise. + + Args: + request (protobuf.Message): The request to send. + """ + if self.call is None: + raise ValueError("Cannot send on an RPC stream that has never been opened.") + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if self.call.is_active(): + self._request_queue.put(request) + else: + # calling next should cause the call to raise. + next(self.call) + + def recv(self): + """Wait for a message to be returned from the stream. + + Recv is blocking. + + If the underlying RPC has been closed, this will raise. + + Returns: + protobuf.Message: The received message. + """ + if self.call is None: + raise ValueError("Cannot recv on an RPC stream that has never been opened.") + + return next(self.call) + + @property + def is_active(self): + """True if this stream is currently open and active.""" + return self.call is not None and self.call.is_active() + + +def _never_terminate(future_or_error): + """By default, no errors cause BiDi termination.""" + return False + + +class ResumableBidiRpc(BidiRpc): + """A :class:`BidiRpc` that can automatically resume the stream on errors. + + It uses the ``should_recover`` arg to determine if it should re-establish + the stream on error. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNAVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + metadata = [('header_name', 'value')] + + rpc = ResumableBidiRpc( + stub.StreamingRpc, + should_recover=should_recover, + initial_request=initial_request, + metadata=metadata + ) + + rpc.open() + + while rpc.is_active(): + print(rpc.recv()) + rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + Args: + start_rpc (grpc.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[protobuf.Message, + Callable[None, protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + should_recover (Callable[[Exception], bool]): A function that returns + True if the stream should be recovered. This will be called + whenever an error is encountered on the stream. + should_terminate (Callable[[Exception], bool]): A function that returns + True if the stream should be terminated. This will be called + whenever an error is encountered on the stream. + metadata Sequence[Tuple(str, str)]: RPC metadata to include in + the request. + throttle_reopen (bool): If ``True``, throttling will be applied to + stream reopen calls. Defaults to ``False``. + """ + + def __init__( + self, + start_rpc, + should_recover, + should_terminate=_never_terminate, + initial_request=None, + metadata=None, + throttle_reopen=False, + ): + super(ResumableBidiRpc, self).__init__(start_rpc, initial_request, metadata) + self._should_recover = should_recover + self._should_terminate = should_terminate + self._operational_lock = threading.RLock() + self._finalized = False + self._finalize_lock = threading.Lock() + + if throttle_reopen: + self._reopen_throttle = _Throttle( + access_limit=5, time_window=datetime.timedelta(seconds=10) + ) + else: + self._reopen_throttle = None + + def _finalize(self, result): + with self._finalize_lock: + if self._finalized: + return + + for callback in self._callbacks: + callback(result) + + self._finalized = True + + def _on_call_done(self, future): + # Unlike the base class, we only execute the callbacks on a terminal + # error, not for errors that we can recover from. Note that grpc's + # "future" here is also a grpc.RpcError. + with self._operational_lock: + if self._should_terminate(future): + self._finalize(future) + elif not self._should_recover(future): + self._finalize(future) + else: + _LOGGER.debug("Re-opening stream from gRPC callback.") + self._reopen() + + def _reopen(self): + with self._operational_lock: + # Another thread already managed to re-open this stream. + if self.call is not None and self.call.is_active(): + _LOGGER.debug("Stream was already re-established.") + return + + self.call = None + # Request generator should exit cleanly since the RPC its bound to + # has exited. + self._request_generator = None + + # Note: we do not currently do any sort of backoff here. The + # assumption is that re-establishing the stream under normal + # circumstances will happen in intervals greater than 60s. + # However, it is possible in a degenerative case that the server + # closes the stream rapidly which would lead to thrashing here, + # but hopefully in those cases the server would return a non- + # retryable error. + + try: + if self._reopen_throttle: + with self._reopen_throttle: + self.open() + else: + self.open() + # If re-opening or re-calling the method fails for any reason, + # consider it a terminal error and finalize the stream. + except Exception as exc: + _LOGGER.debug("Failed to re-open stream due to %s", exc) + self._finalize(exc) + raise + + _LOGGER.info("Re-established stream") + + def _recoverable(self, method, *args, **kwargs): + """Wraps a method to recover the stream and retry on error. + + If a retryable error occurs while making the call, then the stream will + be re-opened and the method will be retried. This happens indefinitely + so long as the error is a retryable one. If an error occurs while + re-opening the stream, then this method will raise immediately and + trigger finalization of this object. + + Args: + method (Callable[..., Any]): The method to call. + args: The args to pass to the method. + kwargs: The kwargs to pass to the method. + """ + while True: + try: + return method(*args, **kwargs) + + except Exception as exc: + with self._operational_lock: + _LOGGER.debug("Call to retryable %r caused %s.", method, exc) + + if self._should_terminate(exc): + self.close() + _LOGGER.debug("Terminating %r due to %s.", method, exc) + self._finalize(exc) + break + + if not self._should_recover(exc): + self.close() + _LOGGER.debug("Not retrying %r due to %s.", method, exc) + self._finalize(exc) + raise exc + + _LOGGER.debug("Re-opening stream from retryable %r.", method) + self._reopen() + + def _send(self, request): + # Grab a reference to the RPC call. Because another thread (notably + # the gRPC error thread) can modify self.call (by invoking reopen), + # we should ensure our reference can not change underneath us. + # If self.call is modified (such as replaced with a new RPC call) then + # this will use the "old" RPC, which should result in the same + # exception passed into gRPC's error handler being raised here, which + # will be handled by the usual error handling in retryable. + with self._operational_lock: + call = self.call + + if call is None: + raise ValueError("Cannot send on an RPC that has never been opened.") + + # Don't use self.is_active(), as ResumableBidiRpc will overload it + # to mean something semantically different. + if call.is_active(): + self._request_queue.put(request) + pass + else: + # calling next should cause the call to raise. + next(call) + + def send(self, request): + return self._recoverable(self._send, request) + + def _recv(self): + with self._operational_lock: + call = self.call + + if call is None: + raise ValueError("Cannot recv on an RPC that has never been opened.") + + return next(call) + + def recv(self): + return self._recoverable(self._recv) + + def close(self): + self._finalize(None) + super(ResumableBidiRpc, self).close() + + @property + def is_active(self): + """bool: True if this stream is currently open and active.""" + # Use the operational lock. It's entirely possible for something + # to check the active state *while* the RPC is being retried. + # Also, use finalized to track the actual terminal state here. + # This is because if the stream is re-established by the gRPC thread + # it's technically possible to check this between when gRPC marks the + # RPC as inactive and when gRPC executes our callback that re-opens + # the stream. + with self._operational_lock: + return self.call is not None and not self._finalized + + +class BackgroundConsumer(object): + """A bi-directional stream consumer that runs in a separate thread. + + This maps the consumption of a stream into a callback-based model. It also + provides :func:`pause` and :func:`resume` to allow for flow-control. + + Example:: + + def should_recover(exc): + return ( + isinstance(exc, grpc.RpcError) and + exc.code() == grpc.StatusCode.UNAVAILABLE) + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + + rpc = ResumeableBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + should_recover=should_recover) + + def on_response(response): + print(response) + + consumer = BackgroundConsumer(rpc, on_response) + consumer.start() + + Note that error handling *must* be done by using the provided + ``bidi_rpc``'s ``add_done_callback``. This helper will automatically exit + whenever the RPC itself exits and will not provide any error details. + + Args: + bidi_rpc (BidiRpc): The RPC to consume. Should not have been + ``open()``ed yet. + on_response (Callable[[protobuf.Message], None]): The callback to + be called for every response on the stream. + on_fatal_exception (Callable[[Exception], None]): The callback to + be called on fatal errors during consumption. Default None. + """ + + def __init__(self, bidi_rpc, on_response, on_fatal_exception=None): + self._bidi_rpc = bidi_rpc + self._on_response = on_response + self._paused = False + self._on_fatal_exception = on_fatal_exception + self._wake = threading.Condition() + self._thread = None + self._operational_lock = threading.Lock() + + def _on_call_done(self, future): + # Resume the thread if it's paused, this prevents blocking forever + # when the RPC has terminated. + self.resume() + + def _thread_main(self, ready): + try: + ready.set() + self._bidi_rpc.add_done_callback(self._on_call_done) + self._bidi_rpc.open() + + while self._bidi_rpc.is_active: + # Do not allow the paused status to change at all during this + # section. There is a condition where we could be resumed + # between checking if we are paused and calling wake.wait(), + # which means that we will miss the notification to wake up + # (oops!) and wait for a notification that will never come. + # Keeping the lock throughout avoids that. + # In the future, we could use `Condition.wait_for` if we drop + # Python 2.7. + # See: https://github.com/googleapis/python-api-core/issues/211 + with self._wake: + while self._paused: + _LOGGER.debug("paused, waiting for waking.") + self._wake.wait() + _LOGGER.debug("woken.") + + _LOGGER.debug("waiting for recv.") + response = self._bidi_rpc.recv() + _LOGGER.debug("recved response.") + if self._on_response is not None: + self._on_response(response) + + except exceptions.GoogleAPICallError as exc: + _LOGGER.debug( + "%s caught error %s and will exit. Generally this is due to " + "the RPC itself being cancelled and the error will be " + "surfaced to the calling code.", + _BIDIRECTIONAL_CONSUMER_NAME, + exc, + exc_info=True, + ) + if self._on_fatal_exception is not None: + self._on_fatal_exception(exc) + + except Exception as exc: + _LOGGER.exception( + "%s caught unexpected exception %s and will exit.", + _BIDIRECTIONAL_CONSUMER_NAME, + exc, + ) + if self._on_fatal_exception is not None: + self._on_fatal_exception(exc) + + _LOGGER.info("%s exiting", _BIDIRECTIONAL_CONSUMER_NAME) + + def start(self): + """Start the background thread and begin consuming the thread.""" + with self._operational_lock: + ready = threading.Event() + thread = threading.Thread( + name=_BIDIRECTIONAL_CONSUMER_NAME, + target=self._thread_main, + args=(ready,), + daemon=True, + ) + thread.start() + # Other parts of the code rely on `thread.is_alive` which + # isn't sufficient to know if a thread is active, just that it may + # soon be active. This can cause races. Further protect + # against races by using a ready event and wait on it to be set. + ready.wait() + self._thread = thread + _LOGGER.debug("Started helper thread %s", thread.name) + + def stop(self): + """Stop consuming the stream and shutdown the background thread. + + NOTE: Cannot be called within `_thread_main`, since it is not + possible to join a thread to itself. + """ + with self._operational_lock: + self._bidi_rpc.close() + + if self._thread is not None: + # Resume the thread to wake it up in case it is sleeping. + self.resume() + # The daemonized thread may itself block, so don't wait + # for it longer than a second. + self._thread.join(1.0) + if self._thread.is_alive(): # pragma: NO COVER + _LOGGER.warning("Background thread did not exit.") + + self._thread = None + self._on_response = None + self._on_fatal_exception = None + + @property + def is_active(self): + """bool: True if the background thread is active.""" + return self._thread is not None and self._thread.is_alive() + + def pause(self): + """Pauses the response stream. + + This does *not* pause the request stream. + """ + with self._wake: + self._paused = True + + def resume(self): + """Resumes the response stream.""" + with self._wake: + self._paused = False + self._wake.notify_all() + + @property + def is_paused(self): + """bool: True if the response stream is paused.""" + return self._paused diff --git a/py311/lib/python3.11/site-packages/google/api_core/bidi_async.py b/py311/lib/python3.11/site-packages/google/api_core/bidi_async.py new file mode 100644 index 0000000000000000000000000000000000000000..3770f69dd6d36d65b56147a5a814bd9b46115bc4 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/bidi_async.py @@ -0,0 +1,244 @@ +# Copyright 2025, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Asynchronous bi-directional streaming RPC helpers.""" + +import asyncio +import logging +from typing import Callable, Optional, Union + +from grpc import aio + +from google.api_core import exceptions +from google.api_core.bidi_base import BidiRpcBase + +from google.protobuf.message import Message as ProtobufMessage + + +_LOGGER = logging.getLogger(__name__) + + +class _AsyncRequestQueueGenerator: + """_AsyncRequestQueueGenerator is a helper class for sending asynchronous + requests to a gRPC stream from a Queue. + + This generator takes asynchronous requests off a given `asyncio.Queue` and + yields them to gRPC. + + It's useful when you have an indeterminate, indefinite, or otherwise + open-ended set of requests to send through a request-streaming (or + bidirectional) RPC. + + Example:: + + requests = _AsyncRequestQueueGenerator(q) + call = await stub.StreamingRequest(requests) + requests.call = call + + async for response in call: + print(response) + await q.put(...) + + Args: + queue (asyncio.Queue): The request queue. + initial_request (Union[ProtobufMessage, + Callable[[], ProtobufMessage]]): The initial request to + yield. This is done independently of the request queue to allow for + easily restarting streams that require some initial configuration + request. + """ + + def __init__( + self, + queue: asyncio.Queue, + initial_request: Optional[ + Union[ProtobufMessage, Callable[[], ProtobufMessage]] + ] = None, + ) -> None: + self._queue = queue + self._initial_request = initial_request + self.call: Optional[aio.Call] = None + + def _is_active(self) -> bool: + """Returns true if the call is not set or not completed.""" + # Note: there is a possibility that this starts *before* the call + # property is set. So we have to check if self.call is set before + # seeing if it's active. We need to return True if self.call is None. + # See https://github.com/googleapis/python-api-core/issues/560. + return self.call is None or not self.call.done() + + async def __aiter__(self): + # The reason this is necessary is because it lets the user have + # control on when they would want to send requests proto messages + # instead of sending all of them initially. + # + # This is achieved via asynchronous queue (asyncio.Queue), + # gRPC awaits until there's a message in the queue. + # + # Finally, it allows for retrying without swapping queues because if + # it does pull an item off the queue when the RPC is inactive, it'll + # immediately put it back and then exit. This is necessary because + # yielding the item in this case will cause gRPC to discard it. In + # practice, this means that the order of messages is not guaranteed. + # If preserving order is necessary it would be easy to use a priority + # queue. + if self._initial_request is not None: + if callable(self._initial_request): + yield self._initial_request() + else: + yield self._initial_request + + while True: + item = await self._queue.get() + + # The consumer explicitly sent "None", indicating that the request + # should end. + if item is None: + _LOGGER.debug("Cleanly exiting request generator.") + return + + if not self._is_active(): + # We have an item, but the call is closed. We should put the + # item back on the queue so that the next call can consume it. + await self._queue.put(item) + _LOGGER.debug( + "Inactive call, replacing item on queue and exiting " + "request generator." + ) + return + + yield item + + +class AsyncBidiRpc(BidiRpcBase): + """A helper for consuming a async bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + Example:: + + initial_request = example_pb2.StreamingRpcRequest( + setting='example') + rpc = AsyncBidiRpc( + stub.StreamingRpc, + initial_request=initial_request, + metadata=[('name', 'value')] + ) + + await rpc.open() + + while rpc.is_active: + print(await rpc.recv()) + await rpc.send(example_pb2.StreamingRpcRequest( + data='example')) + + await rpc.close() + + This does *not* retry the stream on errors. + + Args: + start_rpc (grpc.aio.StreamStreamMultiCallable): The gRPC method used to + start the RPC. + initial_request (Union[ProtobufMessage, + Callable[[], ProtobufMessage]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + metadata (Sequence[Tuple(str, str)]): RPC metadata to include in + the request. + """ + + def _create_queue(self) -> asyncio.Queue: + """Create a queue for requests.""" + return asyncio.Queue() + + async def open(self) -> None: + """Opens the stream.""" + if self.is_active: + raise ValueError("Cannot open an already open stream.") + + request_generator = _AsyncRequestQueueGenerator( + self._request_queue, initial_request=self._initial_request + ) + try: + call = await self._start_rpc(request_generator, metadata=self._rpc_metadata) + except exceptions.GoogleAPICallError as exc: + # The original `grpc.aio.AioRpcError` (which is usually also a + # `grpc.aio.Call`) is available from the ``response`` property on + # the mapped exception. + self._on_call_done(exc.response) + raise + + request_generator.call = call + + # TODO: api_core should expose the future interface for wrapped + # callables as well. + if hasattr(call, "_wrapped"): # pragma: NO COVER + call._wrapped.add_done_callback(self._on_call_done) + else: + call.add_done_callback(self._on_call_done) + + self._request_generator = request_generator + self.call = call + + async def close(self) -> None: + """Closes the stream.""" + if self.call is not None: + self.call.cancel() + + # Put None in request queue to signal termination. + await self._request_queue.put(None) + self._request_generator = None + self._initial_request = None + self._callbacks = [] + # Don't set self.call to None. Keep it around so that send/recv can + # raise the error. + + async def send(self, request: ProtobufMessage) -> None: + """Queue a message to be sent on the stream. + + If the underlying RPC has been closed, this will raise. + + Args: + request (ProtobufMessage): The request to send. + """ + if self.call is None: + raise ValueError("Cannot send on an RPC stream that has never been opened.") + + if not self.call.done(): + await self._request_queue.put(request) + else: + # calling read should cause the call to raise. + await self.call.read() + + async def recv(self) -> ProtobufMessage: + """Wait for a message to be returned from the stream. + + If the underlying RPC has been closed, this will raise. + + Returns: + ProtobufMessage: The received message. + """ + if self.call is None: + raise ValueError("Cannot recv on an RPC stream that has never been opened.") + + return await self.call.read() + + @property + def is_active(self) -> bool: + """Whether the stream is currently open and active.""" + return self.call is not None and not self.call.done() diff --git a/py311/lib/python3.11/site-packages/google/api_core/bidi_base.py b/py311/lib/python3.11/site-packages/google/api_core/bidi_base.py new file mode 100644 index 0000000000000000000000000000000000000000..9288fda41643a0cfdbc239797b95323cbb5c1b7c --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/bidi_base.py @@ -0,0 +1,88 @@ +# Copyright 2025, Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may obtain a copy of the License at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base class for bi-directional streaming RPC helpers.""" + + +class BidiRpcBase: + """A base class for consuming a bi-directional streaming RPC. + + This maps gRPC's built-in interface which uses a request iterator and a + response iterator into a socket-like :func:`send` and :func:`recv`. This + is a more useful pattern for long-running or asymmetric streams (streams + where there is not a direct correlation between the requests and + responses). + + This does *not* retry the stream on errors. + + Args: + start_rpc (Union[grpc.StreamStreamMultiCallable, + grpc.aio.StreamStreamMultiCallable]): The gRPC method used + to start the RPC. + initial_request (Union[protobuf.Message, + Callable[[], protobuf.Message]]): The initial request to + yield. This is useful if an initial request is needed to start the + stream. + metadata (Sequence[Tuple(str, str)]): RPC metadata to include in + the request. + """ + + def __init__(self, start_rpc, initial_request=None, metadata=None): + self._start_rpc = start_rpc + self._initial_request = initial_request + self._rpc_metadata = metadata + self._request_queue = self._create_queue() + self._request_generator = None + self._callbacks = [] + self.call = None + + def _create_queue(self): + """Create a queue for requests.""" + raise NotImplementedError("`_create_queue` is not implemented.") + + def add_done_callback(self, callback): + """Adds a callback that will be called when the RPC terminates. + + This occurs when the RPC errors or is successfully terminated. + + Args: + callback (Union[Callable[[grpc.Future], None], Callable[[Any], None]]): + The callback to execute after gRPC call completed (success or + failure). + + For sync streaming gRPC: Callable[[grpc.Future], None] + + For async streaming gRPC: Callable[[Any], None] + """ + self._callbacks.append(callback) + + def _on_call_done(self, future): + # This occurs when the RPC errors or is successfully terminated. + # Note that grpc's "future" here can also be a grpc.RpcError. + # See note in https://github.com/grpc/grpc/issues/10885#issuecomment-302651331 + # that `grpc.RpcError` is also `grpc.Call`. + # for asynchronous gRPC call it would be `grpc.aio.AioRpcError` + + # Note: sync callbacks can be limiting for async code, because you can't + # await anything in a sync callback. + for callback in self._callbacks: + callback(future) + + @property + def is_active(self): + """True if the gRPC call is not done yet.""" + raise NotImplementedError("`is_active` is not implemented.") + + @property + def pending_requests(self): + """Estimate of the number of queued requests.""" + return self._request_queue.qsize() diff --git a/py311/lib/python3.11/site-packages/google/api_core/client_info.py b/py311/lib/python3.11/site-packages/google/api_core/client_info.py new file mode 100644 index 0000000000000000000000000000000000000000..f0678d24ba1e93e4723bebea5a7ca09ed56fde5e --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/client_info.py @@ -0,0 +1,114 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for providing client information. + +Client information is used to send information about the calling client, +such as the library and Python version, to API services. +""" + +import platform +from typing import Union + +from google.api_core import version as api_core_version + +_PY_VERSION = platform.python_version() +_API_CORE_VERSION = api_core_version.__version__ + +_GRPC_VERSION: Union[str, None] + +try: + import grpc + + _GRPC_VERSION = grpc.__version__ +except ImportError: # pragma: NO COVER + _GRPC_VERSION = None + + +class ClientInfo(object): + """Client information used to generate a user-agent for API calls. + + This user-agent information is sent along with API calls to allow the + receiving service to do analytics on which versions of Python and Google + libraries are being used. + + Args: + python_version (str): The Python interpreter version, for example, + ``'3.9.6'``. + grpc_version (Optional[str]): The gRPC library version. + api_core_version (str): The google-api-core library version. + gapic_version (Optional[str]): The version of gapic-generated client + library, if the library was generated by gapic. + client_library_version (Optional[str]): The version of the client + library, generally used if the client library was not generated + by gapic or if additional functionality was built on top of + a gapic client library. + user_agent (Optional[str]): Prefix to the user agent header. This is + used to supply information such as application name or partner tool. + Recommended format: ``application-or-tool-ID/major.minor.version``. + rest_version (Optional[str]): A string with labeled versions of the + dependencies used for REST transport. + protobuf_runtime_version (Optional[str]): The protobuf runtime version. + """ + + def __init__( + self, + python_version=_PY_VERSION, + grpc_version=_GRPC_VERSION, + api_core_version=_API_CORE_VERSION, + gapic_version=None, + client_library_version=None, + user_agent=None, + rest_version=None, + protobuf_runtime_version=None, + ): + self.python_version = python_version + self.grpc_version = grpc_version + self.api_core_version = api_core_version + self.gapic_version = gapic_version + self.client_library_version = client_library_version + self.user_agent = user_agent + self.rest_version = rest_version + self.protobuf_runtime_version = protobuf_runtime_version + + def to_user_agent(self): + """Returns the user-agent string for this client info.""" + + # Note: the order here is important as the internal metrics system + # expects these items to be in specific locations. + ua = "" + + if self.user_agent is not None: + ua += "{user_agent} " + + ua += "gl-python/{python_version} " + + if self.grpc_version is not None: + ua += "grpc/{grpc_version} " + + if self.rest_version is not None: + ua += "rest/{rest_version} " + + ua += "gax/{api_core_version} " + + if self.gapic_version is not None: + ua += "gapic/{gapic_version} " + + if self.client_library_version is not None: + ua += "gccl/{client_library_version} " + + if self.protobuf_runtime_version is not None: + ua += "pb/{protobuf_runtime_version} " + + return ua.format(**self.__dict__).strip() diff --git a/py311/lib/python3.11/site-packages/google/api_core/client_logging.py b/py311/lib/python3.11/site-packages/google/api_core/client_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..837e3e0c455b4f3dc42c357ac18d379ebaa1e09b --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/client_logging.py @@ -0,0 +1,144 @@ +import logging +import json +import os + +from typing import List, Optional + +_LOGGING_INITIALIZED = False +_BASE_LOGGER_NAME = "google" + +# Fields to be included in the StructuredLogFormatter. +# +# TODO(https://github.com/googleapis/python-api-core/issues/761): Update this list to support additional logging fields. +_recognized_logging_fields = [ + "httpRequest", + "rpcName", + "serviceName", + "credentialsType", + "credentialsInfo", + "universeDomain", + "request", + "response", + "metadata", + "retryAttempt", + "httpResponse", +] # Additional fields to be Logged. + + +def logger_configured(logger) -> bool: + """Determines whether `logger` has non-default configuration + + Args: + logger: The logger to check. + + Returns: + bool: Whether the logger has any non-default configuration. + """ + return ( + logger.handlers != [] or logger.level != logging.NOTSET or not logger.propagate + ) + + +def initialize_logging(): + """Initializes "google" loggers, partly based on the environment variable + + Initializes the "google" logger and any loggers (at the "google" + level or lower) specified by the environment variable + GOOGLE_SDK_PYTHON_LOGGING_SCOPE, as long as none of these loggers + were previously configured. If any such loggers (including the + "google" logger) are initialized, they are set to NOT propagate + log events up to their parent loggers. + + This initialization is executed only once, and hence the + environment variable is only processed the first time this + function is called. + """ + global _LOGGING_INITIALIZED + if _LOGGING_INITIALIZED: + return + scopes = os.getenv("GOOGLE_SDK_PYTHON_LOGGING_SCOPE", "") + setup_logging(scopes) + _LOGGING_INITIALIZED = True + + +def parse_logging_scopes(scopes: Optional[str] = None) -> List[str]: + """Returns a list of logger names. + + Splits the single string of comma-separated logger names into a list of individual logger name strings. + + Args: + scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.) + + Returns: + A list of all the logger names in scopes. + """ + if not scopes: + return [] + # TODO(https://github.com/googleapis/python-api-core/issues/759): check if the namespace is a valid namespace. + # TODO(b/380481951): Support logging multiple scopes. + # TODO(b/380483756): Raise or log a warning for an invalid scope. + namespaces = [scopes] + return namespaces + + +def configure_defaults(logger): + """Configures `logger` to emit structured info to stdout.""" + if not logger_configured(logger): + console_handler = logging.StreamHandler() + logger.setLevel("DEBUG") + logger.propagate = False + formatter = StructuredLogFormatter() + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + +def setup_logging(scopes: str = ""): + """Sets up logging for the specified `scopes`. + + If the loggers specified in `scopes` have not been previously + configured, this will configure them to emit structured log + entries to stdout, and to not propagate their log events to their + parent loggers. Additionally, if the "google" logger (whether it + was specified in `scopes` or not) was not previously configured, + it will also configure it to not propagate log events to the root + logger. + + Args: + scopes: The name of a single logger. (In the future, this will be a comma-separated list of multiple loggers.) + + """ + + # only returns valid logger scopes (namespaces) + # this list has at most one element. + logger_names = parse_logging_scopes(scopes) + + for namespace in logger_names: + # This will either create a module level logger or get the reference of the base logger instantiated above. + logger = logging.getLogger(namespace) + + # Configure default settings. + configure_defaults(logger) + + # disable log propagation at base logger level to the root logger only if a base logger is not already configured via code changes. + base_logger = logging.getLogger(_BASE_LOGGER_NAME) + if not logger_configured(base_logger): + base_logger.propagate = False + + +# TODO(https://github.com/googleapis/python-api-core/issues/763): Expand documentation. +class StructuredLogFormatter(logging.Formatter): + # TODO(https://github.com/googleapis/python-api-core/issues/761): ensure that additional fields such as + # function name, file name, and line no. appear in a log output. + def format(self, record: logging.LogRecord): + log_obj = { + "timestamp": self.formatTime(record), + "severity": record.levelname, + "name": record.name, + "message": record.getMessage(), + } + + for field_name in _recognized_logging_fields: + value = getattr(record, field_name, None) + if value is not None: + log_obj[field_name] = value + return json.dumps(log_obj) diff --git a/py311/lib/python3.11/site-packages/google/api_core/client_options.py b/py311/lib/python3.11/site-packages/google/api_core/client_options.py new file mode 100644 index 0000000000000000000000000000000000000000..30bff48249c3f424d55d4fdc738b28100ef27632 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/client_options.py @@ -0,0 +1,160 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Client options class. + +Client options provide a consistent interface for user options to be defined +across clients. + +You can pass a client options object to a client. + +.. code-block:: python + + from google.api_core.client_options import ClientOptions + from google.cloud.vision_v1 import ImageAnnotatorClient + + def get_client_cert(): + # code to load client certificate and private key. + return client_cert_bytes, client_private_key_bytes + + options = ClientOptions(api_endpoint="foo.googleapis.com", + client_cert_source=get_client_cert) + + client = ImageAnnotatorClient(client_options=options) + +You can also pass a mapping object. + +.. code-block:: python + + from google.cloud.vision_v1 import ImageAnnotatorClient + + client = ImageAnnotatorClient( + client_options={ + "api_endpoint": "foo.googleapis.com", + "client_cert_source" : get_client_cert + }) + + +""" + +from typing import Callable, Mapping, Optional, Sequence, Tuple +import warnings + +from google.api_core import general_helpers + + +class ClientOptions(object): + """Client Options used to set options on clients. + + Args: + api_endpoint (Optional[str]): The desired API endpoint, e.g., + compute.googleapis.com + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A callback + which returns client certificate bytes and private key bytes both in + PEM format. ``client_cert_source`` and ``client_encrypted_cert_source`` + are mutually exclusive. + client_encrypted_cert_source (Optional[Callable[[], Tuple[str, str, bytes]]]): + A callback which returns client certificate file path, encrypted + private key file path, and the passphrase bytes.``client_cert_source`` + and ``client_encrypted_cert_source`` are mutually exclusive. + quota_project_id (Optional[str]): A project name that a client's + quota belongs to. + credentials_file (Optional[str]): Deprecated. A path to a file storing credentials. + ``credentials_file` and ``api_key`` are mutually exclusive. This argument will be + removed in the next major version of `google-api-core`. + + .. warning:: + Important: If you accept a credential configuration (credential JSON/File/Stream) + from an external source for authentication to Google Cloud Platform, you must + validate it before providing it to any Google API or client library. Providing an + unvalidated credential configuration to Google APIs or libraries can compromise + the security of your systems and data. For more information, refer to + `Validate credential configurations from external sources`_. + + .. _Validate credential configurations from external sources: + + https://cloud.google.com/docs/authentication/external/externally-sourced-credentials + scopes (Optional[Sequence[str]]): OAuth access token override scopes. + api_key (Optional[str]): Google API key. ``credentials_file`` and + ``api_key`` are mutually exclusive. + api_audience (Optional[str]): The intended audience for the API calls + to the service that will be set when using certain 3rd party + authentication flows. Audience is typically a resource identifier. + If not set, the service endpoint value will be used as a default. + An example of a valid ``api_audience`` is: "https://language.googleapis.com". + universe_domain (Optional[str]): The desired universe domain. This must match + the one in credentials. If not set, the default universe domain is + `googleapis.com`. If both `api_endpoint` and `universe_domain` are set, + then `api_endpoint` is used as the service endpoint. If `api_endpoint` is + not specified, the format will be `{service}.{universe_domain}`. + + Raises: + ValueError: If both ``client_cert_source`` and ``client_encrypted_cert_source`` + are provided, or both ``credentials_file`` and ``api_key`` are provided. + """ + + def __init__( + self, + api_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + client_encrypted_cert_source: Optional[ + Callable[[], Tuple[str, str, bytes]] + ] = None, + quota_project_id: Optional[str] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + api_key: Optional[str] = None, + api_audience: Optional[str] = None, + universe_domain: Optional[str] = None, + ): + if credentials_file is not None: + warnings.warn(general_helpers._CREDENTIALS_FILE_WARNING, DeprecationWarning) + + if client_cert_source and client_encrypted_cert_source: + raise ValueError( + "client_cert_source and client_encrypted_cert_source are mutually exclusive" + ) + if api_key and credentials_file: + raise ValueError("api_key and credentials_file are mutually exclusive") + self.api_endpoint = api_endpoint + self.client_cert_source = client_cert_source + self.client_encrypted_cert_source = client_encrypted_cert_source + self.quota_project_id = quota_project_id + self.credentials_file = credentials_file + self.scopes = scopes + self.api_key = api_key + self.api_audience = api_audience + self.universe_domain = universe_domain + + def __repr__(self) -> str: + return "ClientOptions: " + repr(self.__dict__) + + +def from_dict(options: Mapping[str, object]) -> ClientOptions: + """Construct a client options object from a mapping object. + + Args: + options (collections.abc.Mapping): A mapping object with client options. + See the docstring for ClientOptions for details on valid arguments. + """ + + client_options = ClientOptions() + + for key, value in options.items(): + if hasattr(client_options, key): + setattr(client_options, key, value) + else: + raise ValueError("ClientOptions does not accept an option '" + key + "'") + + return client_options diff --git a/py311/lib/python3.11/site-packages/google/api_core/datetime_helpers.py b/py311/lib/python3.11/site-packages/google/api_core/datetime_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..c3792300031283f92d91cf221f8ce42e127acd11 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/datetime_helpers.py @@ -0,0 +1,298 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for :mod:`datetime`.""" + +import calendar +import datetime +import re + +from google.protobuf import timestamp_pb2 + + +_UTC_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) +_RFC3339_MICROS = "%Y-%m-%dT%H:%M:%S.%fZ" +_RFC3339_NO_FRACTION = "%Y-%m-%dT%H:%M:%S" +# datetime.strptime cannot handle nanosecond precision: parse w/ regex +_RFC3339_NANOS = re.compile( + r""" + (?P + \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} # YYYY-MM-DDTHH:MM:SS + ) + ( # Optional decimal part + \. # decimal point + (?P\d{1,9}) # nanoseconds, maybe truncated + )? + Z # Zulu +""", + re.VERBOSE, +) + + +def utcnow(): + """A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests.""" + return datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) + + +def to_milliseconds(value): + """Convert a zone-aware datetime to milliseconds since the unix epoch. + + Args: + value (datetime.datetime): The datetime to covert. + + Returns: + int: Milliseconds since the unix epoch. + """ + micros = to_microseconds(value) + return micros // 1000 + + +def from_microseconds(value): + """Convert timestamp in microseconds since the unix epoch to datetime. + + Args: + value (float): The timestamp to convert, in microseconds. + + Returns: + datetime.datetime: The datetime object equivalent to the timestamp in + UTC. + """ + return _UTC_EPOCH + datetime.timedelta(microseconds=value) + + +def to_microseconds(value): + """Convert a datetime to microseconds since the unix epoch. + + Args: + value (datetime.datetime): The datetime to covert. + + Returns: + int: Microseconds since the unix epoch. + """ + if not value.tzinfo: + value = value.replace(tzinfo=datetime.timezone.utc) + # Regardless of what timezone is on the value, convert it to UTC. + value = value.astimezone(datetime.timezone.utc) + # Convert the datetime to a microsecond timestamp. + return int(calendar.timegm(value.timetuple()) * 1e6) + value.microsecond + + +def from_iso8601_date(value): + """Convert a ISO8601 date string to a date. + + Args: + value (str): The ISO8601 date string. + + Returns: + datetime.date: A date equivalent to the date string. + """ + return datetime.datetime.strptime(value, "%Y-%m-%d").date() + + +def from_iso8601_time(value): + """Convert a zoneless ISO8601 time string to a time. + + Args: + value (str): The ISO8601 time string. + + Returns: + datetime.time: A time equivalent to the time string. + """ + return datetime.datetime.strptime(value, "%H:%M:%S").time() + + +def from_rfc3339(value): + """Convert an RFC3339-format timestamp to a native datetime. + + Supported formats include those without fractional seconds, or with + any fraction up to nanosecond precision. + + .. note:: + Python datetimes do not support nanosecond precision; this function + therefore truncates such values to microseconds. + + Args: + value (str): The RFC3339 string to convert. + + Returns: + datetime.datetime: The datetime object equivalent to the timestamp + in UTC. + + Raises: + ValueError: If the timestamp does not match the RFC3339 + regular expression. + """ + with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + raise ValueError( + "Timestamp: {!r}, does not match pattern: {!r}".format( + value, _RFC3339_NANOS.pattern + ) + ) + + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + + if fraction is None: + micros = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10**scale) + micros = nanos // 1000 + + return bare_seconds.replace(microsecond=micros, tzinfo=datetime.timezone.utc) + + +from_rfc3339_nanos = from_rfc3339 # from_rfc3339_nanos method was deprecated. + + +def to_rfc3339(value, ignore_zone=True): + """Convert a datetime to an RFC3339 timestamp string. + + Args: + value (datetime.datetime): + The datetime object to be converted to a string. + ignore_zone (bool): If True, then the timezone (if any) of the + datetime object is ignored and the datetime is treated as UTC. + + Returns: + str: The RFC3339 formatted string representing the datetime. + """ + if not ignore_zone and value.tzinfo is not None: + # Convert to UTC and remove the time zone info. + value = value.replace(tzinfo=None) - value.utcoffset() + + return value.strftime(_RFC3339_MICROS) + + +class DatetimeWithNanoseconds(datetime.datetime): + """Track nanosecond in addition to normal datetime attrs. + + Nanosecond can be passed only as a keyword argument. + """ + + __slots__ = ("_nanosecond",) + + # pylint: disable=arguments-differ + def __new__(cls, *args, **kw): + nanos = kw.pop("nanosecond", 0) + if nanos > 0: + if "microsecond" in kw: + raise TypeError("Specify only one of 'microsecond' or 'nanosecond'") + kw["microsecond"] = nanos // 1000 + inst = datetime.datetime.__new__(cls, *args, **kw) + inst._nanosecond = nanos or 0 + return inst + + # pylint: disable=arguments-differ + + @property + def nanosecond(self): + """Read-only: nanosecond precision.""" + return self._nanosecond + + def rfc3339(self): + """Return an RFC3339-compliant timestamp. + + Returns: + (str): Timestamp string according to RFC3339 spec. + """ + if self._nanosecond == 0: + return to_rfc3339(self) + nanos = str(self._nanosecond).rjust(9, "0").rstrip("0") + return "{}.{}Z".format(self.strftime(_RFC3339_NO_FRACTION), nanos) + + @classmethod + def from_rfc3339(cls, stamp): + """Parse RFC3339-compliant timestamp, preserving nanoseconds. + + Args: + stamp (str): RFC3339 stamp, with up to nanosecond precision + + Returns: + :class:`DatetimeWithNanoseconds`: + an instance matching the timestamp string + + Raises: + ValueError: if `stamp` does not match the expected format + """ + with_nanos = _RFC3339_NANOS.match(stamp) + if with_nanos is None: + raise ValueError( + "Timestamp: {}, does not match pattern: {}".format( + stamp, _RFC3339_NANOS.pattern + ) + ) + bare = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + if fraction is None: + nanos = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10**scale) + return cls( + bare.year, + bare.month, + bare.day, + bare.hour, + bare.minute, + bare.second, + nanosecond=nanos, + tzinfo=datetime.timezone.utc, + ) + + def timestamp_pb(self): + """Return a timestamp message. + + Returns: + (:class:`~google.protobuf.timestamp_pb2.Timestamp`): Timestamp message + """ + inst = ( + self + if self.tzinfo is not None + else self.replace(tzinfo=datetime.timezone.utc) + ) + delta = inst - _UTC_EPOCH + seconds = int(delta.total_seconds()) + nanos = self._nanosecond or self.microsecond * 1000 + return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos) + + @classmethod + def from_timestamp_pb(cls, stamp): + """Parse RFC3339-compliant timestamp, preserving nanoseconds. + + Args: + stamp (:class:`~google.protobuf.timestamp_pb2.Timestamp`): timestamp message + + Returns: + :class:`DatetimeWithNanoseconds`: + an instance matching the timestamp message + """ + microseconds = int(stamp.seconds * 1e6) + bare = from_microseconds(microseconds) + return cls( + bare.year, + bare.month, + bare.day, + bare.hour, + bare.minute, + bare.second, + nanosecond=stamp.nanos, + tzinfo=datetime.timezone.utc, + ) diff --git a/py311/lib/python3.11/site-packages/google/api_core/exceptions.py b/py311/lib/python3.11/site-packages/google/api_core/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..e3eb696c7c5eb0590466635ab749500cd5d77fa5 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/exceptions.py @@ -0,0 +1,670 @@ +# Copyright 2014 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exceptions raised by Google API core & clients. + +This module provides base classes for all errors raised by libraries based +on :mod:`google.api_core`, including both HTTP and gRPC clients. +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import http.client +from typing import Optional, Dict +from typing import Union +import warnings + +from google.rpc import error_details_pb2 + + +def _warn_could_not_import_grpcio_status(): + warnings.warn( + "Please install grpcio-status to obtain helpful grpc error messages.", + ImportWarning, + ) # pragma: NO COVER + + +try: + import grpc + + try: + from grpc_status import rpc_status + except ImportError: # pragma: NO COVER + _warn_could_not_import_grpcio_status() + rpc_status = None +except ImportError: # pragma: NO COVER + grpc = None + +# Lookup tables for mapping exceptions from HTTP and gRPC transports. +# Populated by _GoogleAPICallErrorMeta +_HTTP_CODE_TO_EXCEPTION: Dict[int, Exception] = {} +_GRPC_CODE_TO_EXCEPTION: Dict[int, Exception] = {} + +# Additional lookup table to map integer status codes to grpc status code +# grpc does not currently support initializing enums from ints +# i.e., grpc.StatusCode(5) raises an error +_INT_TO_GRPC_CODE = {} +if grpc is not None: # pragma: no branch + for x in grpc.StatusCode: + _INT_TO_GRPC_CODE[x.value[0]] = x + + +class GoogleAPIError(Exception): + """Base class for all exceptions raised by Google API Clients.""" + + pass + + +class DuplicateCredentialArgs(GoogleAPIError): + """Raised when multiple credentials are passed.""" + + pass + + +class RetryError(GoogleAPIError): + """Raised when a function has exhausted all of its available retries. + + Args: + message (str): The exception message. + cause (Exception): The last exception raised when retrying the + function. + """ + + def __init__(self, message, cause): + super(RetryError, self).__init__(message) + self.message = message + self._cause = cause + + @property + def cause(self): + """The last exception raised when retrying the function.""" + return self._cause + + def __str__(self): + return "{}, last exception: {}".format(self.message, self.cause) + + +class _GoogleAPICallErrorMeta(type): + """Metaclass for registering GoogleAPICallError subclasses.""" + + def __new__(mcs, name, bases, class_dict): + cls = type.__new__(mcs, name, bases, class_dict) + if cls.code is not None: + _HTTP_CODE_TO_EXCEPTION.setdefault(cls.code, cls) + if cls.grpc_status_code is not None: + _GRPC_CODE_TO_EXCEPTION.setdefault(cls.grpc_status_code, cls) + return cls + + +class GoogleAPICallError(GoogleAPIError, metaclass=_GoogleAPICallErrorMeta): + """Base class for exceptions raised by calling API methods. + + Args: + message (str): The exception message. + errors (Sequence[Any]): An optional list of error details. + details (Sequence[Any]): An optional list of objects defined in google.rpc.error_details. + response (Union[requests.Request, grpc.Call]): The response or + gRPC call metadata. + error_info (Union[error_details_pb2.ErrorInfo, None]): An optional object containing error info + (google.rpc.error_details.ErrorInfo). + """ + + code: Union[int, None] = None + """Optional[int]: The HTTP status code associated with this error. + + This may be ``None`` if the exception does not have a direct mapping + to an HTTP error. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html + """ + + grpc_status_code = None + """Optional[grpc.StatusCode]: The gRPC status code associated with this + error. + + This may be ``None`` if the exception does not match up to a gRPC error. + """ + + def __init__(self, message, errors=(), details=(), response=None, error_info=None): + super(GoogleAPICallError, self).__init__(message) + self.message = message + """str: The exception message.""" + self._errors = errors + self._details = details + self._response = response + self._error_info = error_info + + def __str__(self): + error_msg = "{} {}".format(self.code, self.message) + if self.details: + error_msg = "{} {}".format(error_msg, self.details) + # Note: This else condition can be removed once proposal A from + # b/284179390 is implemented. + else: + if self.errors: + errors = [ + f"{error.code}: {error.message}" + for error in self.errors + if hasattr(error, "code") and hasattr(error, "message") + ] + if errors: + error_msg = "{} {}".format(error_msg, "\n".join(errors)) + return error_msg + + @property + def reason(self): + """The reason of the error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing reason of the error. + """ + return self._error_info.reason if self._error_info else None + + @property + def domain(self): + """The logical grouping to which the "reason" belongs. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[str, None]: An optional string containing a logical grouping to which the "reason" belongs. + """ + return self._error_info.domain if self._error_info else None + + @property + def metadata(self): + """Additional structured details about this error. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto#L112 + + Returns: + Union[Dict[str, str], None]: An optional object containing structured details about the error. + """ + return self._error_info.metadata if self._error_info else None + + @property + def errors(self): + """Detailed error information. + + Returns: + Sequence[Any]: A list of additional error details. + """ + return list(self._errors) + + @property + def details(self): + """Information contained in google.rpc.status.details. + + Reference: + https://github.com/googleapis/googleapis/blob/master/google/rpc/status.proto + https://github.com/googleapis/googleapis/blob/master/google/rpc/error_details.proto + + Returns: + Sequence[Any]: A list of structured objects from error_details.proto + """ + return list(self._details) + + @property + def response(self): + """Optional[Union[requests.Request, grpc.Call]]: The response or + gRPC call metadata.""" + return self._response + + +class Redirection(GoogleAPICallError): + """Base class for for all redirection (HTTP 3xx) responses.""" + + +class MovedPermanently(Redirection): + """Exception mapping a ``301 Moved Permanently`` response.""" + + code = http.client.MOVED_PERMANENTLY + + +class NotModified(Redirection): + """Exception mapping a ``304 Not Modified`` response.""" + + code = http.client.NOT_MODIFIED + + +class TemporaryRedirect(Redirection): + """Exception mapping a ``307 Temporary Redirect`` response.""" + + code = http.client.TEMPORARY_REDIRECT + + +class ResumeIncomplete(Redirection): + """Exception mapping a ``308 Resume Incomplete`` response. + + .. note:: :attr:`http.client.PERMANENT_REDIRECT` is ``308``, but Google + APIs differ in their use of this status code. + """ + + code = 308 + + +class ClientError(GoogleAPICallError): + """Base class for all client error (HTTP 4xx) responses.""" + + +class BadRequest(ClientError): + """Exception mapping a ``400 Bad Request`` response.""" + + code = http.client.BAD_REQUEST + + +class InvalidArgument(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.INVALID_ARGUMENT` error.""" + + grpc_status_code = grpc.StatusCode.INVALID_ARGUMENT if grpc is not None else None + + +class FailedPrecondition(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.FAILED_PRECONDITION` + error.""" + + grpc_status_code = grpc.StatusCode.FAILED_PRECONDITION if grpc is not None else None + + +class OutOfRange(BadRequest): + """Exception mapping a :attr:`grpc.StatusCode.OUT_OF_RANGE` error.""" + + grpc_status_code = grpc.StatusCode.OUT_OF_RANGE if grpc is not None else None + + +class Unauthorized(ClientError): + """Exception mapping a ``401 Unauthorized`` response.""" + + code = http.client.UNAUTHORIZED + + +class Unauthenticated(Unauthorized): + """Exception mapping a :attr:`grpc.StatusCode.UNAUTHENTICATED` error.""" + + grpc_status_code = grpc.StatusCode.UNAUTHENTICATED if grpc is not None else None + + +class Forbidden(ClientError): + """Exception mapping a ``403 Forbidden`` response.""" + + code = http.client.FORBIDDEN + + +class PermissionDenied(Forbidden): + """Exception mapping a :attr:`grpc.StatusCode.PERMISSION_DENIED` error.""" + + grpc_status_code = grpc.StatusCode.PERMISSION_DENIED if grpc is not None else None + + +class NotFound(ClientError): + """Exception mapping a ``404 Not Found`` response or a + :attr:`grpc.StatusCode.NOT_FOUND` error.""" + + code = http.client.NOT_FOUND + grpc_status_code = grpc.StatusCode.NOT_FOUND if grpc is not None else None + + +class MethodNotAllowed(ClientError): + """Exception mapping a ``405 Method Not Allowed`` response.""" + + code = http.client.METHOD_NOT_ALLOWED + + +class Conflict(ClientError): + """Exception mapping a ``409 Conflict`` response.""" + + code = http.client.CONFLICT + + +class AlreadyExists(Conflict): + """Exception mapping a :attr:`grpc.StatusCode.ALREADY_EXISTS` error.""" + + grpc_status_code = grpc.StatusCode.ALREADY_EXISTS if grpc is not None else None + + +class Aborted(Conflict): + """Exception mapping a :attr:`grpc.StatusCode.ABORTED` error.""" + + grpc_status_code = grpc.StatusCode.ABORTED if grpc is not None else None + + +class LengthRequired(ClientError): + """Exception mapping a ``411 Length Required`` response.""" + + code = http.client.LENGTH_REQUIRED + + +class PreconditionFailed(ClientError): + """Exception mapping a ``412 Precondition Failed`` response.""" + + code = http.client.PRECONDITION_FAILED + + +class RequestRangeNotSatisfiable(ClientError): + """Exception mapping a ``416 Request Range Not Satisfiable`` response.""" + + code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE + + +class TooManyRequests(ClientError): + """Exception mapping a ``429 Too Many Requests`` response.""" + + code = http.client.TOO_MANY_REQUESTS + + +class ResourceExhausted(TooManyRequests): + """Exception mapping a :attr:`grpc.StatusCode.RESOURCE_EXHAUSTED` error.""" + + grpc_status_code = grpc.StatusCode.RESOURCE_EXHAUSTED if grpc is not None else None + + +class Cancelled(ClientError): + """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error.""" + + # This maps to HTTP status code 499. See + # https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto + code = 499 + grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None + + +class ServerError(GoogleAPICallError): + """Base for 5xx responses.""" + + +class InternalServerError(ServerError): + """Exception mapping a ``500 Internal Server Error`` response. or a + :attr:`grpc.StatusCode.INTERNAL` error.""" + + code = http.client.INTERNAL_SERVER_ERROR + grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None + + +class Unknown(ServerError): + """Exception mapping a :attr:`grpc.StatusCode.UNKNOWN` error.""" + + grpc_status_code = grpc.StatusCode.UNKNOWN if grpc is not None else None + + +class DataLoss(ServerError): + """Exception mapping a :attr:`grpc.StatusCode.DATA_LOSS` error.""" + + grpc_status_code = grpc.StatusCode.DATA_LOSS if grpc is not None else None + + +class MethodNotImplemented(ServerError): + """Exception mapping a ``501 Not Implemented`` response or a + :attr:`grpc.StatusCode.UNIMPLEMENTED` error.""" + + code = http.client.NOT_IMPLEMENTED + grpc_status_code = grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None + + +class BadGateway(ServerError): + """Exception mapping a ``502 Bad Gateway`` response.""" + + code = http.client.BAD_GATEWAY + + +class ServiceUnavailable(ServerError): + """Exception mapping a ``503 Service Unavailable`` response or a + :attr:`grpc.StatusCode.UNAVAILABLE` error.""" + + code = http.client.SERVICE_UNAVAILABLE + grpc_status_code = grpc.StatusCode.UNAVAILABLE if grpc is not None else None + + +class GatewayTimeout(ServerError): + """Exception mapping a ``504 Gateway Timeout`` response.""" + + code = http.client.GATEWAY_TIMEOUT + + +class DeadlineExceeded(GatewayTimeout): + """Exception mapping a :attr:`grpc.StatusCode.DEADLINE_EXCEEDED` error.""" + + grpc_status_code = grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None + + +class AsyncRestUnsupportedParameterError(NotImplementedError): + """Raised when an unsupported parameter is configured against async rest transport.""" + + pass + + +def exception_class_for_http_status(status_code): + """Return the exception class for a specific HTTP status code. + + Args: + status_code (int): The HTTP status code. + + Returns: + :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_http_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from an HTTP status code. + + Args: + status_code (int): The HTTP status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + error_class = exception_class_for_http_status(status_code) + error = error_class(message, **kwargs) + + if error.code is None: + error.code = status_code + + return error + + +def _format_rest_error_message(error, method, url): + method = method.upper() if method else None + message = "{method} {url}: {error}".format( + method=method, + url=url, + error=error, + ) + return message + + +# NOTE: We're moving away from `from_http_status` because it expects an aiohttp response compared +# to `format_http_response_error` which expects a more abstract response from google.auth and is +# compatible with both sync and async response types. +# TODO(https://github.com/googleapis/python-api-core/issues/691): Add type hint for response. +def format_http_response_error( + response, method: str, url: str, payload: Optional[Dict] = None +): + """Create a :class:`GoogleAPICallError` from a google auth rest response. + + Args: + response Union[google.auth.transport.Response, google.auth.aio.transport.Response]: The HTTP response. + method Optional(str): The HTTP request method. + url Optional(str): The HTTP request url. + payload Optional(dict): The HTTP response payload. If not passed in, it is read from response for a response type of google.auth.transport.Response. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`, with the message and errors populated + from the response. + """ + payload = {} if not payload else payload + error_message = payload.get("error", {}).get("message", "unknown error") + errors = payload.get("error", {}).get("errors", ()) + # In JSON, details are already formatted in developer-friendly way. + details = payload.get("error", {}).get("details", ()) + error_info_list = list( + filter( + lambda detail: detail.get("@type", "") + == "type.googleapis.com/google.rpc.ErrorInfo", + details, + ) + ) + error_info = error_info_list[0] if error_info_list else None + message = _format_rest_error_message(error_message, method, url) + + exception = from_http_status( + response.status_code, + message, + errors=errors, + details=details, + response=response, + error_info=error_info, + ) + return exception + + +def from_http_response(response): + """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`. + + Args: + response (requests.Response): The HTTP response. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`, with the message and errors populated + from the response. + """ + try: + payload = response.json() + except ValueError: + payload = {"error": {"message": response.text or "unknown error"}} + return format_http_response_error( + response, response.request.method, response.request.url, payload + ) + + +def exception_class_for_grpc_status(status_code): + """Return the exception class for a specific :class:`grpc.StatusCode`. + + Args: + status_code (grpc.StatusCode): The gRPC status code. + + Returns: + :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`. + """ + return _GRPC_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError) + + +def from_grpc_status(status_code, message, **kwargs): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.StatusCode`. + + Args: + status_code (Union[grpc.StatusCode, int]): The gRPC status code. + message (str): The exception message. + kwargs: Additional arguments passed to the :class:`GoogleAPICallError` + constructor. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + + if isinstance(status_code, int): + status_code = _INT_TO_GRPC_CODE.get(status_code, status_code) + + error_class = exception_class_for_grpc_status(status_code) + error = error_class(message, **kwargs) + + if error.grpc_status_code is None: + error.grpc_status_code = status_code + + return error + + +def _is_informative_grpc_error(rpc_exc): + return hasattr(rpc_exc, "code") and hasattr(rpc_exc, "details") + + +def _parse_grpc_error_details(rpc_exc): + if not rpc_status: # pragma: NO COVER + _warn_could_not_import_grpcio_status() + return [], None + try: + status = rpc_status.from_call(rpc_exc) + except NotImplementedError: # workaround + return [], None + + if not status: + return [], None + + possible_errors = [ + error_details_pb2.BadRequest, + error_details_pb2.PreconditionFailure, + error_details_pb2.QuotaFailure, + error_details_pb2.ErrorInfo, + error_details_pb2.RetryInfo, + error_details_pb2.ResourceInfo, + error_details_pb2.RequestInfo, + error_details_pb2.DebugInfo, + error_details_pb2.Help, + error_details_pb2.LocalizedMessage, + ] + error_info = None + error_details = [] + for detail in status.details: + matched_detail_cls = list( + filter(lambda x: detail.Is(x.DESCRIPTOR), possible_errors) + ) + # If nothing matched, use detail directly. + if len(matched_detail_cls) == 0: + info = detail + else: + info = matched_detail_cls[0]() + detail.Unpack(info) + error_details.append(info) + if isinstance(info, error_details_pb2.ErrorInfo): + error_info = info + return error_details, error_info + + +def from_grpc_error(rpc_exc): + """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`. + + Args: + rpc_exc (grpc.RpcError): The gRPC error. + + Returns: + GoogleAPICallError: An instance of the appropriate subclass of + :class:`GoogleAPICallError`. + """ + # NOTE(lidiz) All gRPC error shares the parent class grpc.RpcError. + # However, check for grpc.RpcError breaks backward compatibility. + if ( + grpc is not None and isinstance(rpc_exc, grpc.Call) + ) or _is_informative_grpc_error(rpc_exc): + details, err_info = _parse_grpc_error_details(rpc_exc) + return from_grpc_status( + rpc_exc.code(), + rpc_exc.details(), + errors=(rpc_exc,), + details=details, + response=rpc_exc, + error_info=err_info, + ) + else: + return GoogleAPICallError(str(rpc_exc), errors=(rpc_exc,), response=rpc_exc) diff --git a/py311/lib/python3.11/site-packages/google/api_core/extended_operation.py b/py311/lib/python3.11/site-packages/google/api_core/extended_operation.py new file mode 100644 index 0000000000000000000000000000000000000000..d474632baeb0617dea1039ec24b527897616c3bd --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/extended_operation.py @@ -0,0 +1,225 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for extended long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operations using :meth:`ExtendedOperation.result`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + extended_operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + extended_operation = my_api_client.long_running_method() + + def my_callback(ex_op): + print(f"Operation {ex_op.name} completed") + + extended_operation.add_done_callback(my_callback) + +""" + +import threading + +from google.api_core import exceptions +from google.api_core.future import polling + + +class ExtendedOperation(polling.PollingFuture): + """An ExtendedOperation future for interacting with a Google API Long-Running Operation. + + Args: + extended_operation (proto.Message): The initial operation. + refresh (Callable[[], type(extended_operation)]): A callable that returns + the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel the operation. + polling Optional(google.api_core.retry.Retry): The configuration used + for polling. This can be used to control how often :meth:`done` + is polled. If the ``timeout`` argument to :meth:`result` is + specified it will override the ``polling.timeout`` property. + retry Optional(google.api_core.retry.Retry): DEPRECATED use ``polling`` + instead. If specified it will override ``polling`` parameter to + maintain backward compatibility. + + Note: Most long-running API methods use google.api_core.operation.Operation + This class is a wrapper for a subset of methods that use alternative + Long-Running Operation (LRO) semantics. + + Note: there is not a concrete type the extended operation must be. + It MUST have fields that correspond to the following, POSSIBLY WITH DIFFERENT NAMES: + * name: str + * status: Union[str, bool, enum.Enum] + * error_code: int + * error_message: str + """ + + def __init__( + self, + extended_operation, + refresh, + cancel, + polling=polling.DEFAULT_POLLING, + **kwargs, + ): + super().__init__(polling=polling, **kwargs) + self._extended_operation = extended_operation + self._refresh = refresh + self._cancel = cancel + # Note: the extended operation does not give a good way to indicate cancellation. + # We make do with manually tracking cancellation and checking for doneness. + self._cancelled = False + self._completion_lock = threading.Lock() + # Invoke in case the operation came back already complete. + self._handle_refreshed_operation() + + # Note: the following four properties MUST be overridden in a subclass + # if, and only if, the fields in the corresponding extended operation message + # have different names. + # + # E.g. we have an extended operation class that looks like + # + # class MyOperation(proto.Message): + # moniker = proto.Field(proto.STRING, number=1) + # status_msg = proto.Field(proto.STRING, number=2) + # optional http_error_code = proto.Field(proto.INT32, number=3) + # optional http_error_msg = proto.Field(proto.STRING, number=4) + # + # the ExtendedOperation subclass would provide property overrides that map + # to these (poorly named) fields. + @property + def name(self): + return self._extended_operation.name + + @property + def status(self): + return self._extended_operation.status + + @property + def error_code(self): + return self._extended_operation.error_code + + @property + def error_message(self): + return self._extended_operation.error_message + + def __getattr__(self, name): + return getattr(self._extended_operation, name) + + def done(self, retry=None): + self._refresh_and_update(retry) + return self._extended_operation.done + + def cancel(self): + if self.done(): + return False + + self._cancel() + self._cancelled = True + return True + + def cancelled(self): + # TODO(dovs): there is not currently a good way to determine whether the + # operation has been cancelled. + # The best we can do is manually keep track of cancellation + # and check for doneness. + if not self._cancelled: + return False + + self._refresh_and_update() + return self._extended_operation.done + + def _refresh_and_update(self, retry=None): + if not self._extended_operation.done: + self._extended_operation = ( + self._refresh(retry=retry) if retry else self._refresh() + ) + self._handle_refreshed_operation() + + def _handle_refreshed_operation(self): + with self._completion_lock: + if not self._extended_operation.done: + return + + if self.error_code and self.error_message: + # Note: `errors` can be removed once proposal A from + # b/284179390 is implemented. + errors = [] + if hasattr(self, "error") and hasattr(self.error, "errors"): + errors = self.error.errors + exception = exceptions.from_http_status( + status_code=self.error_code, + message=self.error_message, + response=self._extended_operation, + errors=errors, + ) + self.set_exception(exception) + elif self.error_code or self.error_message: + exception = exceptions.GoogleAPICallError( + f"Unexpected error {self.error_code}: {self.error_message}" + ) + self.set_exception(exception) + else: + # Extended operations have no payload. + self.set_result(None) + + @classmethod + def make(cls, refresh, cancel, extended_operation, **kwargs): + """ + Return an instantiated ExtendedOperation (or child) that wraps + * a refresh callable + * a cancel callable (can be a no-op) + * an initial result + + .. note:: + It is the caller's responsibility to set up refresh and cancel + with their correct request argument. + The reason for this is that the services that use Extended Operations + have rpcs that look something like the following: + + // service.proto + service MyLongService { + rpc StartLongTask(StartLongTaskRequest) returns (ExtendedOperation) { + option (google.cloud.operation_service) = "CustomOperationService"; + } + } + + service CustomOperationService { + rpc Get(GetOperationRequest) returns (ExtendedOperation) { + option (google.cloud.operation_polling_method) = true; + } + } + + Any info needed for the poll, e.g. a name, path params, etc. + is held in the request, which the initial client method is in a much + better position to make made because the caller made the initial request. + + TL;DR: the caller sets up closures for refresh and cancel that carry + the properly configured requests. + + Args: + refresh (Callable[Optional[Retry]][type(extended_operation)]): A callable that + returns the latest state of the operation. + cancel (Callable[][Any]): A callable that tries to cancel the operation + on a best effort basis. + extended_operation (Any): The initial response of the long running method. + See the docstring for ExtendedOperation.__init__ for requirements on + the type and fields of extended_operation + """ + return cls(extended_operation, refresh, cancel, **kwargs) diff --git a/py311/lib/python3.11/site-packages/google/api_core/general_helpers.py b/py311/lib/python3.11/site-packages/google/api_core/general_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..06282299de4aaa51e8f731c3402da2a4e3e60334 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/general_helpers.py @@ -0,0 +1,52 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This import for backward compatibility only. +from functools import wraps # noqa: F401 pragma: NO COVER + +_CREDENTIALS_FILE_WARNING = """\ +The `credentials_file` argument is deprecated because of a potential security risk. + +The `google.auth.load_credentials_from_file` method does not validate the credential +configuration. The security risk occurs when a credential configuration is accepted +from a source that is not under your control and used without validation on your side. + +If you know that you will be loading credential configurations of a +specific type, it is recommended to use a credential-type-specific +load method. + +This will ensure that an unexpected credential type with potential for +malicious intent is not loaded unintentionally. You might still have to do +validation for certain credential types. Please follow the recommendations +for that method. For example, if you want to load only service accounts, +you can create the service account credentials explicitly: + +``` +from google.cloud.vision_v1 import ImageAnnotatorClient +from google.oauth2 import service_account + +credentials = service_account.Credentials.from_service_account_file(filename) +client = ImageAnnotatorClient(credentials=credentials) +``` + +If you are loading your credential configuration from an untrusted source and have +not mitigated the risks (e.g. by validating the configuration yourself), make +these changes as soon as possible to prevent security risks to your environment. + +Regardless of the method used, it is always your responsibility to validate +configurations received from external sources. + +Refer to https://cloud.google.com/docs/authentication/external/externally-sourced-credentials +for more details. +""" diff --git a/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers.py b/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..430b8ce4884d8c760123efb26867281b4b47f96d --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers.py @@ -0,0 +1,649 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helpers for :mod:`grpc`.""" +import collections +import functools +from typing import Generic, Iterator, Optional, TypeVar +import warnings + +import google.auth +import google.auth.credentials +import google.auth.transport.grpc +import google.auth.transport.requests +import google.protobuf +import grpc + +from google.api_core import exceptions, general_helpers + +PROTOBUF_VERSION = google.protobuf.__version__ + +# The grpcio-gcp package only has support for protobuf < 4 +if PROTOBUF_VERSION[0:2] == "3.": # pragma: NO COVER + try: + import grpc_gcp + + warnings.warn( + """Support for grpcio-gcp is deprecated. This feature will be + removed from `google-api-core` after January 1, 2024. If you need to + continue to use this feature, please pin to a specific version of + `google-api-core`.""", + DeprecationWarning, + ) + HAS_GRPC_GCP = True + except ImportError: + HAS_GRPC_GCP = False +else: + HAS_GRPC_GCP = False + + +# The list of gRPC Callable interfaces that return iterators. +_STREAM_WRAP_CLASSES = (grpc.UnaryStreamMultiCallable, grpc.StreamStreamMultiCallable) + +# denotes the proto response type for grpc calls +P = TypeVar("P") + + +def _patch_callable_name(callable_): + """Fix-up gRPC callable attributes. + + gRPC callable lack the ``__name__`` attribute which causes + :func:`functools.wraps` to error. This adds the attribute if needed. + """ + if not hasattr(callable_, "__name__"): + callable_.__name__ = callable_.__class__.__name__ + + +def _wrap_unary_errors(callable_): + """Map errors for Unary-Unary and Stream-Unary gRPC callables.""" + _patch_callable_name(callable_) + + @functools.wraps(callable_) + def error_remapped_callable(*args, **kwargs): + try: + return callable_(*args, **kwargs) + except grpc.RpcError as exc: + raise exceptions.from_grpc_error(exc) from exc + + return error_remapped_callable + + +class _StreamingResponseIterator(Generic[P], grpc.Call): + def __init__(self, wrapped, prefetch_first_result=True): + self._wrapped = wrapped + + # This iterator is used in a retry context, and returned outside after init. + # gRPC will not throw an exception until the stream is consumed, so we need + # to retrieve the first result, in order to fail, in order to trigger a retry. + try: + if prefetch_first_result: + self._stored_first_result = next(self._wrapped) + except TypeError: + # It is possible the wrapped method isn't an iterable (a grpc.Call + # for instance). If this happens don't store the first result. + pass + except StopIteration: + # ignore stop iteration at this time. This should be handled outside of retry. + pass + + def __iter__(self) -> Iterator[P]: + """This iterator is also an iterable that returns itself.""" + return self + + def __next__(self) -> P: + """Get the next response from the stream. + + Returns: + protobuf.Message: A single response from the stream. + """ + try: + if hasattr(self, "_stored_first_result"): + result = self._stored_first_result + del self._stored_first_result + return result + return next(self._wrapped) + except grpc.RpcError as exc: + # If the stream has already returned data, we cannot recover here. + raise exceptions.from_grpc_error(exc) from exc + + # grpc.Call & grpc.RpcContext interface + + def add_callback(self, callback): + return self._wrapped.add_callback(callback) + + def cancel(self): + return self._wrapped.cancel() + + def code(self): + return self._wrapped.code() + + def details(self): + return self._wrapped.details() + + def initial_metadata(self): + return self._wrapped.initial_metadata() + + def is_active(self): + return self._wrapped.is_active() + + def time_remaining(self): + return self._wrapped.time_remaining() + + def trailing_metadata(self): + return self._wrapped.trailing_metadata() + + +# public type alias denoting the return type of streaming gapic calls +GrpcStream = _StreamingResponseIterator[P] + + +def _wrap_stream_errors(callable_): + """Wrap errors for Unary-Stream and Stream-Stream gRPC callables. + + The callables that return iterators require a bit more logic to re-map + errors when iterating. This wraps both the initial invocation and the + iterator of the return value to re-map errors. + """ + _patch_callable_name(callable_) + + @functools.wraps(callable_) + def error_remapped_callable(*args, **kwargs): + try: + result = callable_(*args, **kwargs) + # Auto-fetching the first result causes PubSub client's streaming pull + # to hang when re-opening the stream, thus we need examine the hacky + # hidden flag to see if pre-fetching is disabled. + # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257 + prefetch_first = getattr(callable_, "_prefetch_first_result_", True) + return _StreamingResponseIterator( + result, prefetch_first_result=prefetch_first + ) + except grpc.RpcError as exc: + raise exceptions.from_grpc_error(exc) from exc + + return error_remapped_callable + + +def wrap_errors(callable_): + """Wrap a gRPC callable and map :class:`grpc.RpcErrors` to friendly error + classes. + + Errors raised by the gRPC callable are mapped to the appropriate + :class:`google.api_core.exceptions.GoogleAPICallError` subclasses. + The original `grpc.RpcError` (which is usually also a `grpc.Call`) is + available from the ``response`` property on the mapped exception. This + is useful for extracting metadata from the original error. + + Args: + callable_ (Callable): A gRPC callable. + + Returns: + Callable: The wrapped gRPC callable. + """ + if isinstance(callable_, _STREAM_WRAP_CLASSES): + return _wrap_stream_errors(callable_) + else: + return _wrap_unary_errors(callable_) + + +def _create_composite_credentials( + credentials=None, + credentials_file=None, + default_scopes=None, + scopes=None, + ssl_credentials=None, + quota_project_id=None, + default_host=None, +): + """Create the composite credentials for secure channels. + + Args: + credentials (google.auth.credentials.Credentials): The credentials. If + not specified, then this function will attempt to ascertain the + credentials from the environment using :func:`google.auth.default`. + credentials_file (str): Deprecated. A file with credentials that can be loaded with + :func:`google.auth.load_credentials_from_file`. This argument is + mutually exclusive with credentials. This argument will be + removed in the next major version of `google-api-core`. + + .. warning:: + Important: If you accept a credential configuration (credential JSON/File/Stream) + from an external source for authentication to Google Cloud Platform, you must + validate it before providing it to any Google API or client library. Providing an + unvalidated credential configuration to Google APIs or libraries can compromise + the security of your systems and data. For more information, refer to + `Validate credential configurations from external sources`_. + + .. _Validate credential configurations from external sources: + + https://cloud.google.com/docs/authentication/external/externally-sourced-credentials + default_scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. + quota_project_id (str): An optional project to use for billing and quota. + default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". + + Returns: + grpc.ChannelCredentials: The composed channel credentials object. + + Raises: + google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. + """ + if credentials_file is not None: + warnings.warn(general_helpers._CREDENTIALS_FILE_WARNING, DeprecationWarning) + + if credentials and credentials_file: + raise exceptions.DuplicateCredentialArgs( + "'credentials' and 'credentials_file' are mutually exclusive." + ) + + if credentials_file: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, scopes=scopes, default_scopes=default_scopes + ) + elif credentials: + credentials = google.auth.credentials.with_scopes_if_required( + credentials, scopes=scopes, default_scopes=default_scopes + ) + else: + credentials, _ = google.auth.default( + scopes=scopes, default_scopes=default_scopes + ) + + if quota_project_id and isinstance( + credentials, google.auth.credentials.CredentialsWithQuotaProject + ): + credentials = credentials.with_quota_project(quota_project_id) + + request = google.auth.transport.requests.Request() + + # Create the metadata plugin for inserting the authorization header. + metadata_plugin = google.auth.transport.grpc.AuthMetadataPlugin( + credentials, + request, + default_host=default_host, + ) + + # Create a set of grpc.CallCredentials using the metadata plugin. + google_auth_credentials = grpc.metadata_call_credentials(metadata_plugin) + + # if `ssl_credentials` is set, use `grpc.composite_channel_credentials` instead of + # `grpc.compute_engine_channel_credentials` as the former supports passing + # `ssl_credentials` via `channel_credentials` which is needed for mTLS. + if ssl_credentials: + # Combine the ssl credentials and the authorization credentials. + # See https://grpc.github.io/grpc/python/grpc.html#grpc.composite_channel_credentials + return grpc.composite_channel_credentials( + ssl_credentials, google_auth_credentials + ) + else: + # Use grpc.compute_engine_channel_credentials in order to support Direct Path. + # See https://grpc.github.io/grpc/python/grpc.html#grpc.compute_engine_channel_credentials + # TODO(https://github.com/googleapis/python-api-core/issues/598): + # Although `grpc.compute_engine_channel_credentials` returns channel credentials + # outside of a Google Compute Engine environment (GCE), we should determine if + # there is a way to reliably detect a GCE environment so that + # `grpc.compute_engine_channel_credentials` is not called outside of GCE. + return grpc.compute_engine_channel_credentials(google_auth_credentials) + + +def create_channel( + target, + credentials=None, + scopes=None, + ssl_credentials=None, + credentials_file=None, + quota_project_id=None, + default_scopes=None, + default_host=None, + compression=None, + attempt_direct_path: Optional[bool] = False, + **kwargs, +): + """Create a secure channel with credentials. + + Args: + target (str): The target service address in the format 'hostname:port'. + credentials (google.auth.credentials.Credentials): The credentials. If + not specified, then this function will attempt to ascertain the + credentials from the environment using :func:`google.auth.default`. + scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. + credentials_file (str): A file with credentials that can be loaded with + :func:`google.auth.load_credentials_from_file`. This argument is + mutually exclusive with credentials. + + .. warning:: + Important: If you accept a credential configuration (credential JSON/File/Stream) + from an external source for authentication to Google Cloud Platform, you must + validate it before providing it to any Google API or client library. Providing an + unvalidated credential configuration to Google APIs or libraries can compromise + the security of your systems and data. For more information, refer to + `Validate credential configurations from external sources`_. + + .. _Validate credential configurations from external sources: + + https://cloud.google.com/docs/authentication/external/externally-sourced-credentials + quota_project_id (str): An optional project to use for billing and quota. + default_scopes (Sequence[str]): Default scopes passed by a Google client + library. Use 'scopes' for user-defined scopes. + default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". + compression (grpc.Compression): An optional value indicating the + compression method to be used over the lifetime of the channel. + attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted + when the request is made. Direct Path is only available within a Google + Compute Engine (GCE) environment and provides a proxyless connection + which increases the available throughput, reduces latency, and increases + reliability. Note: + + - This argument should only be set in a GCE environment and for Services + that are known to support Direct Path. + - If this argument is set outside of GCE, then this request will fail + unless the back-end service happens to have configured fall-back to DNS. + - If the request causes a `ServiceUnavailable` response, it is recommended + that the client repeat the request with `attempt_direct_path` set to + `False` as the Service may not support Direct Path. + - Using `ssl_credentials` with `attempt_direct_path` set to `True` will + result in `ValueError` as this combination is not yet supported. + + kwargs: Additional key-word args passed to + :func:`grpc_gcp.secure_channel` or :func:`grpc.secure_channel`. + Note: `grpc_gcp` is only supported in environments with protobuf < 4.0.0. + + Returns: + grpc.Channel: The created channel. + + Raises: + google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. + ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`. + """ + + # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`, + # raise ValueError as this is not yet supported. + # See https://github.com/googleapis/python-api-core/issues/590 + if ssl_credentials and attempt_direct_path: + raise ValueError("Using ssl_credentials with Direct Path is not supported") + + composite_credentials = _create_composite_credentials( + credentials=credentials, + credentials_file=credentials_file, + default_scopes=default_scopes, + scopes=scopes, + ssl_credentials=ssl_credentials, + quota_project_id=quota_project_id, + default_host=default_host, + ) + + # Note that grpcio-gcp is deprecated + if HAS_GRPC_GCP: # pragma: NO COVER + if compression is not None and compression != grpc.Compression.NoCompression: + warnings.warn( + "The `compression` argument is ignored for grpc_gcp.secure_channel creation.", + DeprecationWarning, + ) + if attempt_direct_path: + warnings.warn( + """The `attempt_direct_path` argument is ignored for grpc_gcp.secure_channel creation.""", + DeprecationWarning, + ) + return grpc_gcp.secure_channel(target, composite_credentials, **kwargs) + + if attempt_direct_path: + target = _modify_target_for_direct_path(target) + + return grpc.secure_channel( + target, composite_credentials, compression=compression, **kwargs + ) + + +def _modify_target_for_direct_path(target: str) -> str: + """ + Given a target, return a modified version which is compatible with Direct Path. + + Args: + target (str): The target service address in the format 'hostname[:port]' or + 'dns://hostname[:port]'. + + Returns: + target (str): The target service address which is converted into a format compatible with Direct Path. + If the target contains `dns:///` or does not contain `:///`, the target will be converted in + a format compatible with Direct Path; otherwise the original target will be returned as the + original target may already denote Direct Path. + """ + + # A DNS prefix may be included with the target to indicate the endpoint is living in the Internet, + # outside of Google Cloud Platform. + dns_prefix = "dns:///" + # Remove "dns:///" if `attempt_direct_path` is set to True as + # the Direct Path prefix `google-c2p:///` will be used instead. + target = target.replace(dns_prefix, "") + + direct_path_separator = ":///" + if direct_path_separator not in target: + target_without_port = target.split(":")[0] + # Modify the target to use Direct Path by adding the `google-c2p:///` prefix + target = f"google-c2p{direct_path_separator}{target_without_port}" + return target + + +_MethodCall = collections.namedtuple( + "_MethodCall", ("request", "timeout", "metadata", "credentials", "compression") +) + +_ChannelRequest = collections.namedtuple("_ChannelRequest", ("method", "request")) + + +class _CallableStub(object): + """Stub for the grpc.*MultiCallable interfaces.""" + + def __init__(self, method, channel): + self._method = method + self._channel = channel + self.response = None + """Union[protobuf.Message, Callable[protobuf.Message], exception]: + The response to give when invoking this callable. If this is a + callable, it will be invoked with the request protobuf. If it's an + exception, the exception will be raised when this is invoked. + """ + self.responses = None + """Iterator[ + Union[protobuf.Message, Callable[protobuf.Message], exception]]: + An iterator of responses. If specified, self.response will be populated + on each invocation by calling ``next(self.responses)``.""" + self.requests = [] + """List[protobuf.Message]: All requests sent to this callable.""" + self.calls = [] + """List[Tuple]: All invocations of this callable. Each tuple is the + request, timeout, metadata, compression, and credentials.""" + + def __call__( + self, request, timeout=None, metadata=None, credentials=None, compression=None + ): + self._channel.requests.append(_ChannelRequest(self._method, request)) + self.calls.append( + _MethodCall(request, timeout, metadata, credentials, compression) + ) + self.requests.append(request) + + response = self.response + if self.responses is not None: + if response is None: + response = next(self.responses) + else: + raise ValueError( + "{method}.response and {method}.responses are mutually " + "exclusive.".format(method=self._method) + ) + + if callable(response): + return response(request) + + if isinstance(response, Exception): + raise response + + if response is not None: + return response + + raise ValueError('Method stub for "{}" has no response.'.format(self._method)) + + +def _simplify_method_name(method): + """Simplifies a gRPC method name. + + When gRPC invokes the channel to create a callable, it gives a full + method name like "/google.pubsub.v1.Publisher/CreateTopic". This + returns just the name of the method, in this case "CreateTopic". + + Args: + method (str): The name of the method. + + Returns: + str: The simplified name of the method. + """ + return method.rsplit("/", 1).pop() + + +class ChannelStub(grpc.Channel): + """A testing stub for the grpc.Channel interface. + + This can be used to test any client that eventually uses a gRPC channel + to communicate. By passing in a channel stub, you can configure which + responses are returned and track which requests are made. + + For example: + + .. code-block:: python + + channel_stub = grpc_helpers.ChannelStub() + client = FooClient(channel=channel_stub) + + channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') + + foo = client.get_foo(labels=['baz']) + + assert foo.name == 'bar' + assert channel_stub.GetFoo.requests[0].labels = ['baz'] + + Each method on the stub can be accessed and configured on the channel. + Here's some examples of various configurations: + + .. code-block:: python + + # Return a basic response: + + channel_stub.GetFoo.response = foo_pb2.Foo(name='bar') + assert client.get_foo().name == 'bar' + + # Raise an exception: + channel_stub.GetFoo.response = NotFound('...') + + with pytest.raises(NotFound): + client.get_foo() + + # Use a sequence of responses: + channel_stub.GetFoo.responses = iter([ + foo_pb2.Foo(name='bar'), + foo_pb2.Foo(name='baz'), + ]) + + assert client.get_foo().name == 'bar' + assert client.get_foo().name == 'baz' + + # Use a callable + + def on_get_foo(request): + return foo_pb2.Foo(name='bar' + request.id) + + channel_stub.GetFoo.response = on_get_foo + + assert client.get_foo(id='123').name == 'bar123' + """ + + def __init__(self, responses=[]): + self.requests = [] + """Sequence[Tuple[str, protobuf.Message]]: A list of all requests made + on this channel in order. The tuple is of method name, request + message.""" + self._method_stubs = {} + + def _stub_for_method(self, method): + method = _simplify_method_name(method) + self._method_stubs[method] = _CallableStub(method, self) + return self._method_stubs[method] + + def __getattr__(self, key): + try: + return self._method_stubs[key] + except KeyError: + raise AttributeError + + def unary_unary( + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, + ): + """grpc.Channel.unary_unary implementation.""" + return self._stub_for_method(method) + + def unary_stream( + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, + ): + """grpc.Channel.unary_stream implementation.""" + return self._stub_for_method(method) + + def stream_unary( + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, + ): + """grpc.Channel.stream_unary implementation.""" + return self._stub_for_method(method) + + def stream_stream( + self, + method, + request_serializer=None, + response_deserializer=None, + _registered_method=False, + ): + """grpc.Channel.stream_stream implementation.""" + return self._stub_for_method(method) + + def subscribe(self, callback, try_to_connect=False): + """grpc.Channel.subscribe implementation.""" + pass + + def unsubscribe(self, callback): + """grpc.Channel.unsubscribe implementation.""" + pass + + def close(self): + """grpc.Channel.close implementation.""" + pass diff --git a/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers_async.py b/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers_async.py new file mode 100644 index 0000000000000000000000000000000000000000..9e1ad1105d45cd9863ba53ae881dbcb3d213f888 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/grpc_helpers_async.py @@ -0,0 +1,348 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AsyncIO helpers for :mod:`grpc` supporting 3.7+. + +Please combine more detailed docstring in grpc_helpers.py to use following +functions. This module is implementing the same surface with AsyncIO semantics. +""" + +import asyncio +import functools +import warnings + +from typing import AsyncGenerator, Generic, Iterator, Optional, TypeVar + +import grpc +from grpc import aio + +from google.api_core import exceptions, general_helpers, grpc_helpers + +# denotes the proto response type for grpc calls +P = TypeVar("P") + +# NOTE(lidiz) Alternatively, we can hack "__getattribute__" to perform +# automatic patching for us. But that means the overhead of creating an +# extra Python function spreads to every single send and receive. + + +class _WrappedCall(aio.Call): + def __init__(self): + self._call = None + + def with_call(self, call): + """Supplies the call object separately to keep __init__ clean.""" + self._call = call + return self + + async def initial_metadata(self): + return await self._call.initial_metadata() + + async def trailing_metadata(self): + return await self._call.trailing_metadata() + + async def code(self): + return await self._call.code() + + async def details(self): + return await self._call.details() + + def cancelled(self): + return self._call.cancelled() + + def done(self): + return self._call.done() + + def time_remaining(self): + return self._call.time_remaining() + + def cancel(self): + return self._call.cancel() + + def add_done_callback(self, callback): + self._call.add_done_callback(callback) + + async def wait_for_connection(self): + try: + await self._call.wait_for_connection() + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + +class _WrappedUnaryResponseMixin(Generic[P], _WrappedCall): + def __await__(self) -> Iterator[P]: + try: + response = yield from self._call.__await__() + return response + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + +class _WrappedStreamResponseMixin(Generic[P], _WrappedCall): + def __init__(self): + self._wrapped_async_generator = None + + async def read(self) -> P: + try: + return await self._call.read() + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + async def _wrapped_aiter(self) -> AsyncGenerator[P, None]: + try: + # NOTE(lidiz) coverage doesn't understand the exception raised from + # __anext__ method. It is covered by test case: + # test_wrap_stream_errors_aiter_non_rpc_error + async for response in self._call: # pragma: no branch + yield response + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + def __aiter__(self) -> AsyncGenerator[P, None]: + if not self._wrapped_async_generator: + self._wrapped_async_generator = self._wrapped_aiter() + return self._wrapped_async_generator + + +class _WrappedStreamRequestMixin(_WrappedCall): + async def write(self, request): + try: + await self._call.write(request) + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + async def done_writing(self): + try: + await self._call.done_writing() + except grpc.RpcError as rpc_error: + raise exceptions.from_grpc_error(rpc_error) from rpc_error + + +# NOTE(lidiz) Implementing each individual class separately, so we don't +# expose any API that should not be seen. E.g., __aiter__ in unary-unary +# RPC, or __await__ in stream-stream RPC. +class _WrappedUnaryUnaryCall(_WrappedUnaryResponseMixin[P], aio.UnaryUnaryCall): + """Wrapped UnaryUnaryCall to map exceptions.""" + + +class _WrappedUnaryStreamCall(_WrappedStreamResponseMixin[P], aio.UnaryStreamCall): + """Wrapped UnaryStreamCall to map exceptions.""" + + +class _WrappedStreamUnaryCall( + _WrappedUnaryResponseMixin[P], _WrappedStreamRequestMixin, aio.StreamUnaryCall +): + """Wrapped StreamUnaryCall to map exceptions.""" + + +class _WrappedStreamStreamCall( + _WrappedStreamRequestMixin, _WrappedStreamResponseMixin[P], aio.StreamStreamCall +): + """Wrapped StreamStreamCall to map exceptions.""" + + +# public type alias denoting the return type of async streaming gapic calls +GrpcAsyncStream = _WrappedStreamResponseMixin +# public type alias denoting the return type of unary gapic calls +AwaitableGrpcCall = _WrappedUnaryResponseMixin + + +def _wrap_unary_errors(callable_): + """Map errors for Unary-Unary async callables.""" + + @functools.wraps(callable_) + def error_remapped_callable(*args, **kwargs): + call = callable_(*args, **kwargs) + return _WrappedUnaryUnaryCall().with_call(call) + + return error_remapped_callable + + +def _wrap_stream_errors(callable_, wrapper_type): + """Map errors for streaming RPC async callables.""" + + @functools.wraps(callable_) + async def error_remapped_callable(*args, **kwargs): + call = callable_(*args, **kwargs) + call = wrapper_type().with_call(call) + await call.wait_for_connection() + return call + + return error_remapped_callable + + +def wrap_errors(callable_): + """Wrap a gRPC async callable and map :class:`grpc.RpcErrors` to + friendly error classes. + + Errors raised by the gRPC callable are mapped to the appropriate + :class:`google.api_core.exceptions.GoogleAPICallError` subclasses. The + original `grpc.RpcError` (which is usually also a `grpc.Call`) is + available from the ``response`` property on the mapped exception. This + is useful for extracting metadata from the original error. + + Args: + callable_ (Callable): A gRPC callable. + + Returns: Callable: The wrapped gRPC callable. + """ + grpc_helpers._patch_callable_name(callable_) + + if isinstance(callable_, aio.UnaryStreamMultiCallable): + return _wrap_stream_errors(callable_, _WrappedUnaryStreamCall) + elif isinstance(callable_, aio.StreamUnaryMultiCallable): + return _wrap_stream_errors(callable_, _WrappedStreamUnaryCall) + elif isinstance(callable_, aio.StreamStreamMultiCallable): + return _wrap_stream_errors(callable_, _WrappedStreamStreamCall) + else: + return _wrap_unary_errors(callable_) + + +def create_channel( + target, + credentials=None, + scopes=None, + ssl_credentials=None, + credentials_file=None, + quota_project_id=None, + default_scopes=None, + default_host=None, + compression=None, + attempt_direct_path: Optional[bool] = False, + **kwargs, +): + """Create an AsyncIO secure channel with credentials. + + Args: + target (str): The target service address in the format 'hostname:port'. + credentials (google.auth.credentials.Credentials): The credentials. If + not specified, then this function will attempt to ascertain the + credentials from the environment using :func:`google.auth.default`. + scopes (Sequence[str]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + ssl_credentials (grpc.ChannelCredentials): Optional SSL channel + credentials. This can be used to specify different certificates. + credentials_file (str): Deprecated. A file with credentials that can be loaded with + :func:`google.auth.load_credentials_from_file`. This argument is + mutually exclusive with credentials. This argument will be + removed in the next major version of `google-api-core`. + + .. warning:: + Important: If you accept a credential configuration (credential JSON/File/Stream) + from an external source for authentication to Google Cloud Platform, you must + validate it before providing it to any Google API or client library. Providing an + unvalidated credential configuration to Google APIs or libraries can compromise + the security of your systems and data. For more information, refer to + `Validate credential configurations from external sources`_. + + .. _Validate credential configurations from external sources: + + https://cloud.google.com/docs/authentication/external/externally-sourced-credentials + quota_project_id (str): An optional project to use for billing and quota. + default_scopes (Sequence[str]): Default scopes passed by a Google client + library. Use 'scopes' for user-defined scopes. + default_host (str): The default endpoint. e.g., "pubsub.googleapis.com". + compression (grpc.Compression): An optional value indicating the + compression method to be used over the lifetime of the channel. + attempt_direct_path (Optional[bool]): If set, Direct Path will be attempted + when the request is made. Direct Path is only available within a Google + Compute Engine (GCE) environment and provides a proxyless connection + which increases the available throughput, reduces latency, and increases + reliability. Note: + + - This argument should only be set in a GCE environment and for Services + that are known to support Direct Path. + - If this argument is set outside of GCE, then this request will fail + unless the back-end service happens to have configured fall-back to DNS. + - If the request causes a `ServiceUnavailable` response, it is recommended + that the client repeat the request with `attempt_direct_path` set to + `False` as the Service may not support Direct Path. + - Using `ssl_credentials` with `attempt_direct_path` set to `True` will + result in `ValueError` as this combination is not yet supported. + + kwargs: Additional key-word args passed to :func:`aio.secure_channel`. + + Returns: + aio.Channel: The created channel. + + Raises: + google.api_core.DuplicateCredentialArgs: If both a credentials object and credentials_file are passed. + ValueError: If `ssl_credentials` is set and `attempt_direct_path` is set to `True`. + """ + + if credentials_file is not None: + warnings.warn(general_helpers._CREDENTIALS_FILE_WARNING, DeprecationWarning) + + # If `ssl_credentials` is set and `attempt_direct_path` is set to `True`, + # raise ValueError as this is not yet supported. + # See https://github.com/googleapis/python-api-core/issues/590 + if ssl_credentials and attempt_direct_path: + raise ValueError("Using ssl_credentials with Direct Path is not supported") + + composite_credentials = grpc_helpers._create_composite_credentials( + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + default_scopes=default_scopes, + ssl_credentials=ssl_credentials, + quota_project_id=quota_project_id, + default_host=default_host, + ) + + if attempt_direct_path: + target = grpc_helpers._modify_target_for_direct_path(target) + + return aio.secure_channel( + target, composite_credentials, compression=compression, **kwargs + ) + + +class FakeUnaryUnaryCall(_WrappedUnaryUnaryCall): + """Fake implementation for unary-unary RPCs. + + It is a dummy object for response message. Supply the intended response + upon the initialization, and the coroutine will return the exact response + message. + """ + + def __init__(self, response=object()): + self.response = response + self._future = asyncio.get_event_loop().create_future() + self._future.set_result(self.response) + + def __await__(self): + response = yield from self._future.__await__() + return response + + +class FakeStreamUnaryCall(_WrappedStreamUnaryCall): + """Fake implementation for stream-unary RPCs. + + It is a dummy object for response message. Supply the intended response + upon the initialization, and the coroutine will return the exact response + message. + """ + + def __init__(self, response=object()): + self.response = response + self._future = asyncio.get_event_loop().create_future() + self._future.set_result(self.response) + + def __await__(self): + response = yield from self._future.__await__() + return response + + async def wait_for_connection(self): + pass diff --git a/py311/lib/python3.11/site-packages/google/api_core/iam.py b/py311/lib/python3.11/site-packages/google/api_core/iam.py new file mode 100644 index 0000000000000000000000000000000000000000..4437c701f09d433062ff413f2a597801e06e2f14 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/iam.py @@ -0,0 +1,427 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Non-API-specific IAM policy definitions + +For allowed roles / permissions, see: +https://cloud.google.com/iam/docs/understanding-roles + +Example usage: + +.. code-block:: python + + # ``get_iam_policy`` returns a :class:'~google.api_core.iam.Policy`. + policy = resource.get_iam_policy(requested_policy_version=3) + + phred = "user:phred@example.com" + admin_group = "group:admins@groups.example.com" + account = "serviceAccount:account-1234@accounts.example.com" + + policy.version = 3 + policy.bindings = [ + { + "role": "roles/owner", + "members": {phred, admin_group, account} + }, + { + "role": "roles/editor", + "members": {"allAuthenticatedUsers"} + }, + { + "role": "roles/viewer", + "members": {"allUsers"} + "condition": { + "title": "request_time", + "description": "Requests made before 2021-01-01T00:00:00Z", + "expression": "request.time < timestamp(\"2021-01-01T00:00:00Z\")" + } + } + ] + + resource.set_iam_policy(policy) +""" + +import collections +import collections.abc +import operator +import warnings + +# Generic IAM roles + +OWNER_ROLE = "roles/owner" +"""Generic role implying all rights to an object.""" + +EDITOR_ROLE = "roles/editor" +"""Generic role implying rights to modify an object.""" + +VIEWER_ROLE = "roles/viewer" +"""Generic role implying rights to access an object.""" + +_ASSIGNMENT_DEPRECATED_MSG = """\ +Assigning to '{}' is deprecated. Use the `policy.bindings` property to modify bindings instead.""" + +_DICT_ACCESS_MSG = """\ +Dict access is not supported on policies with version > 1 or with conditional bindings.""" + + +class InvalidOperationException(Exception): + """Raised when trying to use Policy class as a dict.""" + + pass + + +class Policy(collections.abc.MutableMapping): + """IAM Policy + + Args: + etag (Optional[str]): ETag used to identify a unique of the policy + version (Optional[int]): The syntax schema version of the policy. + + Note: + Using conditions in bindings requires the policy's version to be set + to `3` or greater, depending on the versions that are currently supported. + + Accessing the policy using dict operations will raise InvalidOperationException + when the policy's version is set to 3. + + Use the policy.bindings getter/setter to retrieve and modify the policy's bindings. + + See: + IAM Policy https://cloud.google.com/iam/reference/rest/v1/Policy + Policy versions https://cloud.google.com/iam/docs/policies#versions + Conditions overview https://cloud.google.com/iam/docs/conditions-overview. + """ + + _OWNER_ROLES = (OWNER_ROLE,) + """Roles mapped onto our ``owners`` attribute.""" + + _EDITOR_ROLES = (EDITOR_ROLE,) + """Roles mapped onto our ``editors`` attribute.""" + + _VIEWER_ROLES = (VIEWER_ROLE,) + """Roles mapped onto our ``viewers`` attribute.""" + + def __init__(self, etag=None, version=None): + self.etag = etag + self.version = version + self._bindings = [] + + def __iter__(self): + self.__check_version__() + # Exclude bindings with no members + return (binding["role"] for binding in self._bindings if binding["members"]) + + def __len__(self): + self.__check_version__() + # Exclude bindings with no members + return len(list(self.__iter__())) + + def __getitem__(self, key): + self.__check_version__() + for b in self._bindings: + if b["role"] == key: + return b["members"] + # If the binding does not yet exist, create one + # NOTE: This will create bindings with no members + # which are ignored by __iter__ and __len__ + new_binding = {"role": key, "members": set()} + self._bindings.append(new_binding) + return new_binding["members"] + + def __setitem__(self, key, value): + self.__check_version__() + value = set(value) + for binding in self._bindings: + if binding["role"] == key: + binding["members"] = value + return + self._bindings.append({"role": key, "members": value}) + + def __delitem__(self, key): + self.__check_version__() + for b in self._bindings: + if b["role"] == key: + self._bindings.remove(b) + return + raise KeyError(key) + + def __check_version__(self): + """Raise InvalidOperationException if version is greater than 1 or policy contains conditions.""" + raise_version = self.version is not None and self.version > 1 + + if raise_version or self._contains_conditions(): + raise InvalidOperationException(_DICT_ACCESS_MSG) + + def _contains_conditions(self): + for b in self._bindings: + if b.get("condition") is not None: + return True + return False + + @property + def bindings(self): + """The policy's list of bindings. + + A binding is specified by a dictionary with keys: + + * role (str): Role that is assigned to `members`. + + * members (:obj:`set` of str): Specifies the identities associated to this binding. + + * condition (:obj:`dict` of str:str): Specifies a condition under which this binding will apply. + + * title (str): Title for the condition. + + * description (:obj:str, optional): Description of the condition. + + * expression: A CEL expression. + + Type: + :obj:`list` of :obj:`dict` + + See: + Policy versions https://cloud.google.com/iam/docs/policies#versions + Conditions overview https://cloud.google.com/iam/docs/conditions-overview. + + Example: + + .. code-block:: python + + USER = "user:phred@example.com" + ADMIN_GROUP = "group:admins@groups.example.com" + SERVICE_ACCOUNT = "serviceAccount:account-1234@accounts.example.com" + CONDITION = { + "title": "request_time", + "description": "Requests made before 2021-01-01T00:00:00Z", # Optional + "expression": "request.time < timestamp(\"2021-01-01T00:00:00Z\")" + } + + # Set policy's version to 3 before setting bindings containing conditions. + policy.version = 3 + + policy.bindings = [ + { + "role": "roles/viewer", + "members": {USER, ADMIN_GROUP, SERVICE_ACCOUNT}, + "condition": CONDITION + }, + ... + ] + """ + return self._bindings + + @bindings.setter + def bindings(self, bindings): + self._bindings = bindings + + @property + def owners(self): + """Legacy access to owner role. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to access bindings instead. + """ + result = set() + for role in self._OWNER_ROLES: + for member in self.get(role, ()): + result.add(member) + return frozenset(result) + + @owners.setter + def owners(self, value): + """Update owners. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to access bindings instead. + """ + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("owners", OWNER_ROLE), DeprecationWarning + ) + self[OWNER_ROLE] = value + + @property + def editors(self): + """Legacy access to editor role. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to access bindings instead. + """ + result = set() + for role in self._EDITOR_ROLES: + for member in self.get(role, ()): + result.add(member) + return frozenset(result) + + @editors.setter + def editors(self, value): + """Update editors. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to modify bindings instead. + """ + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("editors", EDITOR_ROLE), + DeprecationWarning, + ) + self[EDITOR_ROLE] = value + + @property + def viewers(self): + """Legacy access to viewer role. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to modify bindings instead. + """ + result = set() + for role in self._VIEWER_ROLES: + for member in self.get(role, ()): + result.add(member) + return frozenset(result) + + @viewers.setter + def viewers(self, value): + """Update viewers. + + Raise InvalidOperationException if version is greater than 1 or policy contains conditions. + + DEPRECATED: use `policy.bindings` to modify bindings instead. + """ + warnings.warn( + _ASSIGNMENT_DEPRECATED_MSG.format("viewers", VIEWER_ROLE), + DeprecationWarning, + ) + self[VIEWER_ROLE] = value + + @staticmethod + def user(email): + """Factory method for a user member. + + Args: + email (str): E-mail for this particular user. + + Returns: + str: A member string corresponding to the given user. + """ + return "user:%s" % (email,) + + @staticmethod + def service_account(email): + """Factory method for a service account member. + + Args: + email (str): E-mail for this particular service account. + + Returns: + str: A member string corresponding to the given service account. + + """ + return "serviceAccount:%s" % (email,) + + @staticmethod + def group(email): + """Factory method for a group member. + + Args: + email (str): An id or e-mail for this particular group. + + Returns: + str: A member string corresponding to the given group. + """ + return "group:%s" % (email,) + + @staticmethod + def domain(domain): + """Factory method for a domain member. + + Args: + domain (str): The domain for this member. + + Returns: + str: A member string corresponding to the given domain. + """ + return "domain:%s" % (domain,) + + @staticmethod + def all_users(): + """Factory method for a member representing all users. + + Returns: + str: A member string representing all users. + """ + return "allUsers" + + @staticmethod + def authenticated_users(): + """Factory method for a member representing all authenticated users. + + Returns: + str: A member string representing all authenticated users. + """ + return "allAuthenticatedUsers" + + @classmethod + def from_api_repr(cls, resource): + """Factory: create a policy from a JSON resource. + + Args: + resource (dict): policy resource returned by ``getIamPolicy`` API. + + Returns: + :class:`Policy`: the parsed policy + """ + version = resource.get("version") + etag = resource.get("etag") + policy = cls(etag, version) + policy.bindings = resource.get("bindings", []) + + for binding in policy.bindings: + binding["members"] = set(binding.get("members", ())) + + return policy + + def to_api_repr(self): + """Render a JSON policy resource. + + Returns: + dict: a resource to be passed to the ``setIamPolicy`` API. + """ + resource = {} + + if self.etag is not None: + resource["etag"] = self.etag + + if self.version is not None: + resource["version"] = self.version + + if self._bindings and len(self._bindings) > 0: + bindings = [] + for binding in self._bindings: + members = binding.get("members") + if members: + new_binding = {"role": binding["role"], "members": sorted(members)} + condition = binding.get("condition") + if condition: + new_binding["condition"] = condition + bindings.append(new_binding) + + if bindings: + # Sort bindings by role + key = operator.itemgetter("role") + resource["bindings"] = sorted(bindings, key=key) + + return resource diff --git a/py311/lib/python3.11/site-packages/google/api_core/operation.py b/py311/lib/python3.11/site-packages/google/api_core/operation.py new file mode 100644 index 0000000000000000000000000000000000000000..5206243a74637394902b43b4b01277fc5e4a0b4f --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/operation.py @@ -0,0 +1,365 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Futures for long-running operations returned from Google Cloud APIs. + +These futures can be used to synchronously wait for the result of a +long-running operation using :meth:`Operation.result`: + + +.. code-block:: python + + operation = my_api_client.long_running_method() + result = operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + operation = my_api_client.long_running_method() + + def my_callback(future): + result = future.result() + + operation.add_done_callback(my_callback) + +""" + +import functools +import threading + +from google.api_core import exceptions +from google.api_core import protobuf_helpers +from google.api_core.future import polling +from google.longrunning import operations_pb2 +from google.protobuf import json_format +from google.rpc import code_pb2 + + +class Operation(polling.PollingFuture): + """A Future for interacting with a Google API Long-Running Operation. + + Args: + operation (google.longrunning.operations_pb2.Operation): The + initial operation. + refresh (Callable[[], ~.api_core.operation.Operation]): A callable that + returns the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel + the operation. + result_type (func:`type`): The protobuf type for the operation's + result. + metadata_type (func:`type`): The protobuf type for the operation's + metadata. + polling (google.api_core.retry.Retry): The configuration used for polling. + This parameter controls how often :meth:`done` is polled. If the + ``timeout`` argument is specified in the :meth:`result` method, it will + override the ``polling.timeout`` property. + retry (google.api_core.retry.Retry): DEPRECATED: use ``polling`` instead. + If specified it will override ``polling`` parameter to maintain + backward compatibility. + """ + + def __init__( + self, + operation, + refresh, + cancel, + result_type, + metadata_type=None, + polling=polling.DEFAULT_POLLING, + **kwargs, + ): + super(Operation, self).__init__(polling=polling, **kwargs) + self._operation = operation + self._refresh = refresh + self._cancel = cancel + self._result_type = result_type + self._metadata_type = metadata_type + self._completion_lock = threading.Lock() + # Invoke this in case the operation came back already complete. + self._set_result_from_operation() + + @property + def operation(self): + """google.longrunning.Operation: The current long-running operation.""" + return self._operation + + @property + def metadata(self): + """google.protobuf.Message: the current operation metadata.""" + if not self._operation.HasField("metadata"): + return None + + return protobuf_helpers.from_any_pb( + self._metadata_type, self._operation.metadata + ) + + @classmethod + def deserialize(self, payload): + """Deserialize a ``google.longrunning.Operation`` protocol buffer. + + Args: + payload (bytes): A serialized operation protocol buffer. + + Returns: + ~.operations_pb2.Operation: An Operation protobuf object. + """ + return operations_pb2.Operation.FromString(payload) + + def _set_result_from_operation(self): + """Set the result or exception from the operation if it is complete.""" + # This must be done in a lock to prevent the polling thread + # and main thread from both executing the completion logic + # at the same time. + with self._completion_lock: + # If the operation isn't complete or if the result has already been + # set, do not call set_result/set_exception again. + # Note: self._result_set is set to True in set_result and + # set_exception, in case those methods are invoked directly. + if not self._operation.done or self._result_set: + return + + if self._operation.HasField("response"): + response = protobuf_helpers.from_any_pb( + self._result_type, self._operation.response + ) + self.set_result(response) + elif self._operation.HasField("error"): + exception = exceptions.from_grpc_status( + status_code=self._operation.error.code, + message=self._operation.error.message, + errors=(self._operation.error,), + response=self._operation, + ) + self.set_exception(exception) + else: + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) + + def _refresh_and_update(self, retry=None): + """Refresh the operation and update the result if needed. + + Args: + retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. + """ + # If the currently cached operation is done, no need to make another + # RPC as it will not change once done. + if not self._operation.done: + self._operation = self._refresh(retry=retry) if retry else self._refresh() + self._set_result_from_operation() + + def done(self, retry=None): + """Checks to see if the operation is complete. + + Args: + retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. + + Returns: + bool: True if the operation is complete, False otherwise. + """ + self._refresh_and_update(retry) + return self._operation.done + + def cancel(self): + """Attempt to cancel the operation. + + Returns: + bool: True if the cancel RPC was made, False if the operation is + already complete. + """ + if self.done(): + return False + + self._cancel() + return True + + def cancelled(self): + """True if the operation was cancelled.""" + self._refresh_and_update() + return ( + self._operation.HasField("error") + and self._operation.error.code == code_pb2.CANCELLED + ) + + +def _refresh_http(api_request, operation_name, retry=None): + """Refresh an operation using a JSON/HTTP client. + + Args: + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + operation_name (str): The name of the operation. + retry (google.api_core.retry.Retry): (Optional) retry policy + + Returns: + google.longrunning.operations_pb2.Operation: The operation. + """ + path = "operations/{}".format(operation_name) + + if retry is not None: + api_request = retry(api_request) + + api_response = api_request(method="GET", path=path) + return json_format.ParseDict(api_response, operations_pb2.Operation()) + + +def _cancel_http(api_request, operation_name): + """Cancel an operation using a JSON/HTTP client. + + Args: + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + operation_name (str): The name of the operation. + """ + path = "operations/{}:cancel".format(operation_name) + api_request(method="POST", path=path) + + +def from_http_json(operation, api_request, result_type, **kwargs): + """Create an operation future using a HTTP/JSON client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via `HTTP/JSON`_. + + .. _HTTP/JSON: https://cloud.google.com/speech/reference/rest/\ + v1beta1/operations#Operation + + Args: + operation (dict): Operation as a dictionary. + api_request (Callable): A callable used to make an API request. This + should generally be + :meth:`google.cloud._http.Connection.api_request`. + result_type (:func:`type`): The protobuf result type. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + operation_proto = json_format.ParseDict(operation, operations_pb2.Operation()) + refresh = functools.partial(_refresh_http, api_request, operation_proto.name) + cancel = functools.partial(_cancel_http, api_request, operation_proto.name) + return Operation(operation_proto, refresh, cancel, result_type, **kwargs) + + +def _refresh_grpc(operations_stub, operation_name, retry=None): + """Refresh an operation using a gRPC client. + + Args: + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The gRPC operations stub. + operation_name (str): The name of the operation. + retry (google.api_core.retry.Retry): (Optional) retry policy + + Returns: + google.longrunning.operations_pb2.Operation: The operation. + """ + request_pb = operations_pb2.GetOperationRequest(name=operation_name) + + rpc = operations_stub.GetOperation + if retry is not None: + rpc = retry(rpc) + + return rpc(request_pb) + + +def _cancel_grpc(operations_stub, operation_name): + """Cancel an operation using a gRPC client. + + Args: + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The gRPC operations stub. + operation_name (str): The name of the operation. + """ + request_pb = operations_pb2.CancelOperationRequest(name=operation_name) + operations_stub.CancelOperation(request_pb) + + +def from_grpc(operation, operations_stub, result_type, grpc_metadata=None, **kwargs): + """Create an operation future using a gRPC client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via gRPC. + + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + + Args: + operation (google.longrunning.operations_pb2.Operation): The operation. + operations_stub (google.longrunning.operations_pb2.OperationsStub): + The operations stub. + result_type (:func:`type`): The protobuf result type. + grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass + to the rpc. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + refresh = functools.partial( + _refresh_grpc, + operations_stub, + operation.name, + metadata=grpc_metadata, + ) + cancel = functools.partial( + _cancel_grpc, + operations_stub, + operation.name, + metadata=grpc_metadata, + ) + return Operation(operation, refresh, cancel, result_type, **kwargs) + + +def from_gapic(operation, operations_client, result_type, grpc_metadata=None, **kwargs): + """Create an operation future from a gapic client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via a gapic client. + + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + + Args: + operation (google.longrunning.operations_pb2.Operation): The operation. + operations_client (google.api_core.operations_v1.OperationsClient): + The operations client. + result_type (:func:`type`): The protobuf result type. + grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass + to the rpc. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + refresh = functools.partial( + operations_client.get_operation, + operation.name, + metadata=grpc_metadata, + ) + cancel = functools.partial( + operations_client.cancel_operation, + operation.name, + metadata=grpc_metadata, + ) + return Operation(operation, refresh, cancel, result_type, **kwargs) diff --git a/py311/lib/python3.11/site-packages/google/api_core/operation_async.py b/py311/lib/python3.11/site-packages/google/api_core/operation_async.py new file mode 100644 index 0000000000000000000000000000000000000000..2fd341d9747921c405404d3d58355c88aeaac08e --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/operation_async.py @@ -0,0 +1,225 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AsyncIO futures for long-running operations returned from Google Cloud APIs. + +These futures can be used to await for the result of a long-running operation +using :meth:`AsyncOperation.result`: + + +.. code-block:: python + + operation = my_api_client.long_running_method() + result = await operation.result() + +Or asynchronously using callbacks and :meth:`Operation.add_done_callback`: + +.. code-block:: python + + operation = my_api_client.long_running_method() + + def my_callback(future): + result = await future.result() + + operation.add_done_callback(my_callback) + +""" + +import functools +import threading + +from google.api_core import exceptions +from google.api_core import protobuf_helpers +from google.api_core.future import async_future +from google.longrunning import operations_pb2 +from google.rpc import code_pb2 + + +class AsyncOperation(async_future.AsyncFuture): + """A Future for interacting with a Google API Long-Running Operation. + + Args: + operation (google.longrunning.operations_pb2.Operation): The + initial operation. + refresh (Callable[[], ~.api_core.operation.Operation]): A callable that + returns the latest state of the operation. + cancel (Callable[[], None]): A callable that tries to cancel + the operation. + result_type (func:`type`): The protobuf type for the operation's + result. + metadata_type (func:`type`): The protobuf type for the operation's + metadata. + retry (google.api_core.retry.Retry): The retry configuration used + when polling. This can be used to control how often :meth:`done` + is polled. Regardless of the retry's ``deadline``, it will be + overridden by the ``timeout`` argument to :meth:`result`. + """ + + def __init__( + self, + operation, + refresh, + cancel, + result_type, + metadata_type=None, + retry=async_future.DEFAULT_RETRY, + ): + super().__init__(retry=retry) + self._operation = operation + self._refresh = refresh + self._cancel = cancel + self._result_type = result_type + self._metadata_type = metadata_type + self._completion_lock = threading.Lock() + # Invoke this in case the operation came back already complete. + self._set_result_from_operation() + + @property + def operation(self): + """google.longrunning.Operation: The current long-running operation.""" + return self._operation + + @property + def metadata(self): + """google.protobuf.Message: the current operation metadata.""" + if not self._operation.HasField("metadata"): + return None + + return protobuf_helpers.from_any_pb( + self._metadata_type, self._operation.metadata + ) + + @classmethod + def deserialize(cls, payload): + """Deserialize a ``google.longrunning.Operation`` protocol buffer. + + Args: + payload (bytes): A serialized operation protocol buffer. + + Returns: + ~.operations_pb2.Operation: An Operation protobuf object. + """ + return operations_pb2.Operation.FromString(payload) + + def _set_result_from_operation(self): + """Set the result or exception from the operation if it is complete.""" + # This must be done in a lock to prevent the async_future thread + # and main thread from both executing the completion logic + # at the same time. + with self._completion_lock: + # If the operation isn't complete or if the result has already been + # set, do not call set_result/set_exception again. + if not self._operation.done or self._future.done(): + return + + if self._operation.HasField("response"): + response = protobuf_helpers.from_any_pb( + self._result_type, self._operation.response + ) + self.set_result(response) + elif self._operation.HasField("error"): + exception = exceptions.GoogleAPICallError( + self._operation.error.message, + errors=(self._operation.error,), + response=self._operation, + ) + self.set_exception(exception) + else: + exception = exceptions.GoogleAPICallError( + "Unexpected state: Long-running operation had neither " + "response nor error set." + ) + self.set_exception(exception) + + async def _refresh_and_update(self, retry=async_future.DEFAULT_RETRY): + """Refresh the operation and update the result if needed. + + Args: + retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. + """ + # If the currently cached operation is done, no need to make another + # RPC as it will not change once done. + if not self._operation.done: + self._operation = await self._refresh(retry=retry) + self._set_result_from_operation() + + async def done(self, retry=async_future.DEFAULT_RETRY): + """Checks to see if the operation is complete. + + Args: + retry (google.api_core.retry.Retry): (Optional) How to retry the RPC. + + Returns: + bool: True if the operation is complete, False otherwise. + """ + await self._refresh_and_update(retry) + return self._operation.done + + async def cancel(self): + """Attempt to cancel the operation. + + Returns: + bool: True if the cancel RPC was made, False if the operation is + already complete. + """ + result = await self.done() + if result: + return False + else: + await self._cancel() + return True + + async def cancelled(self): + """True if the operation was cancelled.""" + await self._refresh_and_update() + return ( + self._operation.HasField("error") + and self._operation.error.code == code_pb2.CANCELLED + ) + + +def from_gapic(operation, operations_client, result_type, grpc_metadata=None, **kwargs): + """Create an operation future from a gapic client. + + This interacts with the long-running operations `service`_ (specific + to a given API) via a gapic client. + + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + + Args: + operation (google.longrunning.operations_pb2.Operation): The operation. + operations_client (google.api_core.operations_v1.OperationsClient): + The operations client. + result_type (:func:`type`): The protobuf result type. + grpc_metadata (Optional[List[Tuple[str, str]]]): Additional metadata to pass + to the rpc. + kwargs: Keyword args passed into the :class:`Operation` constructor. + + Returns: + ~.api_core.operation.Operation: The operation future to track the given + operation. + """ + refresh = functools.partial( + operations_client.get_operation, + operation.name, + metadata=grpc_metadata, + ) + cancel = functools.partial( + operations_client.cancel_operation, + operation.name, + metadata=grpc_metadata, + ) + return AsyncOperation(operation, refresh, cancel, result_type, **kwargs) diff --git a/py311/lib/python3.11/site-packages/google/api_core/page_iterator.py b/py311/lib/python3.11/site-packages/google/api_core/page_iterator.py new file mode 100644 index 0000000000000000000000000000000000000000..23761ec4596247e92ad958f966ad53a97a171a01 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/page_iterator.py @@ -0,0 +1,571 @@ +# Copyright 2015 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Iterators for paging through paged API methods. + +These iterators simplify the process of paging through API responses +where the request takes a page token and the response is a list of results with +a token for the next page. See `list pagination`_ in the Google API Style Guide +for more details. + +.. _list pagination: + https://cloud.google.com/apis/design/design_patterns#list_pagination + +API clients that have methods that follow the list pagination pattern can +return an :class:`.Iterator`. You can use this iterator to get **all** of +the results across all pages:: + + >>> results_iterator = client.list_resources() + >>> list(results_iterator) # Convert to a list (consumes all values). + +Or you can walk your way through items and call off the search early if +you find what you're looking for (resulting in possibly fewer requests):: + + >>> for resource in results_iterator: + ... print(resource.name) + ... if not resource.is_valid: + ... break + +At any point, you may check the number of items consumed by referencing the +``num_results`` property of the iterator:: + + >>> for my_item in results_iterator: + ... if results_iterator.num_results >= 10: + ... break + +When iterating, not every new item will send a request to the server. +To iterate based on each page of items (where a page corresponds to +a request):: + + >>> for page in results_iterator.pages: + ... print('=' * 20) + ... print(' Page number: {:d}'.format(iterator.page_number)) + ... print(' Items in page: {:d}'.format(page.num_items)) + ... print(' First item: {!r}'.format(next(page))) + ... print('Items remaining: {:d}'.format(page.remaining)) + ... print('Next page token: {}'.format(iterator.next_page_token)) + ==================== + Page number: 1 + Items in page: 1 + First item: + Items remaining: 0 + Next page token: eav1OzQB0OM8rLdGXOEsyQWSG + ==================== + Page number: 2 + Items in page: 19 + First item: + Items remaining: 18 + Next page token: None + +Then, for each page you can get all the resources on that page by iterating +through it or using :func:`list`:: + + >>> list(page) + [ + , + , + , + ] +""" + +import abc + + +class Page(object): + """Single page of results in an iterator. + + Args: + parent (google.api_core.page_iterator.Iterator): The iterator that owns + the current page. + items (Sequence[Any]): An iterable (that also defines __len__) of items + from a raw API response. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the raw API response + into the native object. Will be called with the iterator and a + single item. + raw_page Optional[google.protobuf.message.Message]: + The raw page response. + """ + + def __init__(self, parent, items, item_to_value, raw_page=None): + self._parent = parent + self._num_items = len(items) + self._remaining = self._num_items + self._item_iter = iter(items) + self._item_to_value = item_to_value + self._raw_page = raw_page + + @property + def raw_page(self): + """google.protobuf.message.Message""" + return self._raw_page + + @property + def num_items(self): + """int: Total items in the page.""" + return self._num_items + + @property + def remaining(self): + """int: Remaining items in the page.""" + return self._remaining + + def __iter__(self): + """The :class:`Page` is an iterator of items.""" + return self + + def __next__(self): + """Get the next value in the page.""" + item = next(self._item_iter) + result = self._item_to_value(self._parent, item) + # Since we've successfully got the next value from the + # iterator, we update the number of remaining. + self._remaining -= 1 + return result + + +def _item_to_value_identity(iterator, item): + """An item to value transformer that returns the item un-changed.""" + # pylint: disable=unused-argument + # We are conforming to the interface defined by Iterator. + return item + + +class Iterator(object, metaclass=abc.ABCMeta): + """A generic class for iterating through API list responses. + + Args: + client(google.cloud.client.Client): The API client. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the raw API response + into the native object. Will be called with the iterator and a + single item. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. + """ + + def __init__( + self, + client, + item_to_value=_item_to_value_identity, + page_token=None, + max_results=None, + ): + self._started = False + self.__active_iterator = None + + self.client = client + """Optional[Any]: The client that created this iterator.""" + self.item_to_value = item_to_value + """Callable[Iterator, Any]: Callable to convert an item from the type + in the raw API response into the native object. Will be called with + the iterator and a + single item. + """ + self.max_results = max_results + """int: The maximum number of results to fetch""" + + # The attributes below will change over the life of the iterator. + self.page_number = 0 + """int: The current page of results.""" + self.next_page_token = page_token + """str: The token for the next page of results. If this is set before + the iterator starts, it effectively offsets the iterator to a + specific starting point.""" + self.num_results = 0 + """int: The total number of results fetched so far.""" + + @property + def pages(self): + """Iterator of pages in the response. + + returns: + types.GeneratorType[google.api_core.page_iterator.Page]: A + generator of page instances. + + raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._page_iter(increment=True) + + def _items_iter(self): + """Iterator for each item returned.""" + for page in self._page_iter(increment=False): + for item in page: + self.num_results += 1 + yield item + + def __iter__(self): + """Iterator for each item returned. + + Returns: + types.GeneratorType[Any]: A generator of items from the API. + + Raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._items_iter() + + def __next__(self): + if self.__active_iterator is None: + self.__active_iterator = iter(self) + return next(self.__active_iterator) + + def _page_iter(self, increment): + """Generator of pages of API responses. + + Args: + increment (bool): Flag indicating if the total number of results + should be incremented on each page. This is useful since a page + iterator will want to increment by results per page while an + items iterator will want to increment per item. + + Yields: + Page: each page of items from the API. + """ + page = self._next_page() + while page is not None: + self.page_number += 1 + if increment: + self.num_results += page.num_items + yield page + page = self._next_page() + + @abc.abstractmethod + def _next_page(self): + """Get the next page in the iterator. + + This does nothing and is intended to be over-ridden by subclasses + to return the next :class:`Page`. + + Raises: + NotImplementedError: Always, this method is abstract. + """ + raise NotImplementedError + + +def _do_nothing_page_start(iterator, page, response): + """Helper to provide custom behavior after a :class:`Page` is started. + + This is a do-nothing stand-in as the default value. + + Args: + iterator (Iterator): An iterator that holds some request info. + page (Page): The page that was just created. + response (Any): The API response for a page. + """ + # pylint: disable=unused-argument + pass + + +class HTTPIterator(Iterator): + """A generic class for iterating through HTTP/JSON API list responses. + + To make an iterator work, you'll need to provide a way to convert a JSON + item returned from the API into the object of your choice (via + ``item_to_value``). You also may need to specify a custom ``items_key`` so + that a given response (containing a page of results) can be parsed into an + iterable page of the actual objects you want. + + Args: + client (google.cloud.client.Client): The API client. + api_request (Callable): The function to use to make API requests. + Generally, this will be + :meth:`google.cloud._http.JSONConnection.api_request`. + path (str): The method path to query for the list of items. + item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]): + Callable to convert an item from the type in the JSON response into + a native object. Will be called with the iterator and a single + item. + items_key (str): The key in the API response where the list of items + can be found. + page_token (str): A token identifying a page in a result set to start + fetching results from. + page_size (int): The maximum number of results to fetch per page + max_results (int): The maximum number of results to fetch + extra_params (dict): Extra query string parameters for the + API call. + page_start (Callable[ + google.api_core.page_iterator.Iterator, + google.api_core.page_iterator.Page, dict]): Callable to provide + any special behavior after a new page has been created. Assumed + signature takes the :class:`.Iterator` that started the page, + the :class:`.Page` that was started and the dictionary containing + the page response. + next_token (str): The name of the field used in the response for page + tokens. + + .. autoattribute:: pages + """ + + _DEFAULT_ITEMS_KEY = "items" + _PAGE_TOKEN = "pageToken" + _MAX_RESULTS = "maxResults" + _NEXT_TOKEN = "nextPageToken" + _RESERVED_PARAMS = frozenset([_PAGE_TOKEN]) + _HTTP_METHOD = "GET" + + def __init__( + self, + client, + api_request, + path, + item_to_value, + items_key=_DEFAULT_ITEMS_KEY, + page_token=None, + page_size=None, + max_results=None, + extra_params=None, + page_start=_do_nothing_page_start, + next_token=_NEXT_TOKEN, + ): + super(HTTPIterator, self).__init__( + client, item_to_value, page_token=page_token, max_results=max_results + ) + self.api_request = api_request + self.path = path + self._items_key = items_key + self.extra_params = extra_params + self._page_size = page_size + self._page_start = page_start + self._next_token = next_token + # Verify inputs / provide defaults. + if self.extra_params is None: + self.extra_params = {} + self._verify_params() + + def _verify_params(self): + """Verifies the parameters don't use any reserved parameter. + + Raises: + ValueError: If a reserved parameter is used. + """ + reserved_in_use = self._RESERVED_PARAMS.intersection(self.extra_params) + if reserved_in_use: + raise ValueError("Using a reserved parameter", reserved_in_use) + + def _next_page(self): + """Get the next page in the iterator. + + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. + """ + if self._has_next_page(): + response = self._get_next_page_response() + items = response.get(self._items_key, ()) + page = Page(self, items, self.item_to_value, raw_page=response) + self._page_start(self, page, response) + self.next_page_token = response.get(self._next_token) + return page + else: + return None + + def _has_next_page(self): + """Determines whether or not there are more pages with results. + + Returns: + bool: Whether the iterator has more pages. + """ + if self.page_number == 0: + return True + + if self.max_results is not None: + if self.num_results >= self.max_results: + return False + + return self.next_page_token is not None + + def _get_query_params(self): + """Getter for query parameters for the next request. + + Returns: + dict: A dictionary of query parameters. + """ + result = {} + if self.next_page_token is not None: + result[self._PAGE_TOKEN] = self.next_page_token + + page_size = None + if self.max_results is not None: + page_size = self.max_results - self.num_results + if self._page_size is not None: + page_size = min(page_size, self._page_size) + elif self._page_size is not None: + page_size = self._page_size + + if page_size is not None: + result[self._MAX_RESULTS] = page_size + + result.update(self.extra_params) + return result + + def _get_next_page_response(self): + """Requests the next page from the path provided. + + Returns: + dict: The parsed JSON response of the next page's contents. + + Raises: + ValueError: If the HTTP method is not ``GET`` or ``POST``. + """ + params = self._get_query_params() + if self._HTTP_METHOD == "GET": + return self.api_request( + method=self._HTTP_METHOD, path=self.path, query_params=params + ) + elif self._HTTP_METHOD == "POST": + return self.api_request( + method=self._HTTP_METHOD, path=self.path, data=params + ) + else: + raise ValueError("Unexpected HTTP method", self._HTTP_METHOD) + + +class _GAXIterator(Iterator): + """A generic class for iterating through Cloud gRPC APIs list responses. + + Any: + client (google.cloud.client.Client): The API client. + page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped + to conform to the :class:`Iterator` interface. + item_to_value (Callable[Iterator, Any]): Callable to convert an item + from the protobuf response into a native object. Will + be called with the iterator and a single item. + max_results (int): The maximum number of results to fetch. + + .. autoattribute:: pages + """ + + def __init__(self, client, page_iter, item_to_value, max_results=None): + super(_GAXIterator, self).__init__( + client, + item_to_value, + page_token=page_iter.page_token, + max_results=max_results, + ) + self._gax_page_iter = page_iter + + def _next_page(self): + """Get the next page in the iterator. + + Wraps the response from the :class:`~google.gax.PageIterator` in a + :class:`Page` instance and captures some state at each page. + + Returns: + Optional[Page]: The next page in the iterator or :data:`None` if + there are no pages left. + """ + try: + items = next(self._gax_page_iter) + page = Page(self, items, self.item_to_value) + self.next_page_token = self._gax_page_iter.page_token or None + return page + except StopIteration: + return None + + +class GRPCIterator(Iterator): + """A generic class for iterating through gRPC list responses. + + .. note:: The class does not take a ``page_token`` argument because it can + just be specified in the ``request``. + + Args: + client (google.cloud.client.Client): The API client. This unused by + this class, but kept to satisfy the :class:`Iterator` interface. + method (Callable[protobuf.Message]): A bound gRPC method that should + take a single message for the request. + request (protobuf.Message): The request message. + items_field (str): The field in the response message that has the + items for the page. + item_to_value (Callable[GRPCIterator, Any]): Callable to convert an + item from the type in the JSON response into a native object. Will + be called with the iterator and a single item. + request_token_field (str): The field in the request message used to + specify the page token. + response_token_field (str): The field in the response message that has + the token for the next page. + max_results (int): The maximum number of results to fetch. + + .. autoattribute:: pages + """ + + _DEFAULT_REQUEST_TOKEN_FIELD = "page_token" + _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token" + + def __init__( + self, + client, + method, + request, + items_field, + item_to_value=_item_to_value_identity, + request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD, + response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD, + max_results=None, + ): + super(GRPCIterator, self).__init__( + client, item_to_value, max_results=max_results + ) + self._method = method + self._request = request + self._items_field = items_field + self._request_token_field = request_token_field + self._response_token_field = response_token_field + + def _next_page(self): + """Get the next page in the iterator. + + Returns: + Page: The next page in the iterator or :data:`None` if + there are no pages left. + """ + if not self._has_next_page(): + return None + + if self.next_page_token is not None: + setattr(self._request, self._request_token_field, self.next_page_token) + + response = self._method(self._request) + + self.next_page_token = getattr(response, self._response_token_field) + items = getattr(response, self._items_field) + page = Page(self, items, self.item_to_value, raw_page=response) + + return page + + def _has_next_page(self): + """Determines whether or not there are more pages with results. + + Returns: + bool: Whether the iterator has more pages. + """ + if self.page_number == 0: + return True + + if self.max_results is not None: + if self.num_results >= self.max_results: + return False + + # Note: intentionally a falsy check instead of a None check. The RPC + # can return an empty string indicating no more pages. + return True if self.next_page_token else False diff --git a/py311/lib/python3.11/site-packages/google/api_core/page_iterator_async.py b/py311/lib/python3.11/site-packages/google/api_core/page_iterator_async.py new file mode 100644 index 0000000000000000000000000000000000000000..c0725758ec7299dd1bb8a45bcb0299769cff3e45 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/page_iterator_async.py @@ -0,0 +1,285 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AsyncIO iterators for paging through paged API methods. + +These iterators simplify the process of paging through API responses +where the request takes a page token and the response is a list of results with +a token for the next page. See `list pagination`_ in the Google API Style Guide +for more details. + +.. _list pagination: + https://cloud.google.com/apis/design/design_patterns#list_pagination + +API clients that have methods that follow the list pagination pattern can +return an :class:`.AsyncIterator`: + + >>> results_iterator = await client.list_resources() + +Or you can walk your way through items and call off the search early if +you find what you're looking for (resulting in possibly fewer requests):: + + >>> async for resource in results_iterator: + ... print(resource.name) + ... if not resource.is_valid: + ... break + +At any point, you may check the number of items consumed by referencing the +``num_results`` property of the iterator:: + + >>> async for my_item in results_iterator: + ... if results_iterator.num_results >= 10: + ... break + +When iterating, not every new item will send a request to the server. +To iterate based on each page of items (where a page corresponds to +a request):: + + >>> async for page in results_iterator.pages: + ... print('=' * 20) + ... print(' Page number: {:d}'.format(iterator.page_number)) + ... print(' Items in page: {:d}'.format(page.num_items)) + ... print(' First item: {!r}'.format(next(page))) + ... print('Items remaining: {:d}'.format(page.remaining)) + ... print('Next page token: {}'.format(iterator.next_page_token)) + ==================== + Page number: 1 + Items in page: 1 + First item: + Items remaining: 0 + Next page token: eav1OzQB0OM8rLdGXOEsyQWSG + ==================== + Page number: 2 + Items in page: 19 + First item: + Items remaining: 18 + Next page token: None +""" + +import abc + +from google.api_core.page_iterator import Page + + +def _item_to_value_identity(iterator, item): + """An item to value transformer that returns the item un-changed.""" + # pylint: disable=unused-argument + # We are conforming to the interface defined by Iterator. + return item + + +class AsyncIterator(abc.ABC): + """A generic class for iterating through API list responses. + + Args: + client(google.cloud.client.Client): The API client. + item_to_value (Callable[google.api_core.page_iterator_async.AsyncIterator, Any]): + Callable to convert an item from the type in the raw API response + into the native object. Will be called with the iterator and a + single item. + page_token (str): A token identifying a page in a result set to start + fetching results from. + max_results (int): The maximum number of results to fetch. + """ + + def __init__( + self, + client, + item_to_value=_item_to_value_identity, + page_token=None, + max_results=None, + ): + self._started = False + self.__active_aiterator = None + + self.client = client + """Optional[Any]: The client that created this iterator.""" + self.item_to_value = item_to_value + """Callable[Iterator, Any]: Callable to convert an item from the type + in the raw API response into the native object. Will be called with + the iterator and a + single item. + """ + self.max_results = max_results + """int: The maximum number of results to fetch.""" + + # The attributes below will change over the life of the iterator. + self.page_number = 0 + """int: The current page of results.""" + self.next_page_token = page_token + """str: The token for the next page of results. If this is set before + the iterator starts, it effectively offsets the iterator to a + specific starting point.""" + self.num_results = 0 + """int: The total number of results fetched so far.""" + + @property + def pages(self): + """Iterator of pages in the response. + + returns: + types.GeneratorType[google.api_core.page_iterator.Page]: A + generator of page instances. + + raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._page_aiter(increment=True) + + async def _items_aiter(self): + """Iterator for each item returned.""" + async for page in self._page_aiter(increment=False): + for item in page: + self.num_results += 1 + yield item + + def __aiter__(self): + """Iterator for each item returned. + + Returns: + types.GeneratorType[Any]: A generator of items from the API. + + Raises: + ValueError: If the iterator has already been started. + """ + if self._started: + raise ValueError("Iterator has already started", self) + self._started = True + return self._items_aiter() + + async def __anext__(self): + if self.__active_aiterator is None: + self.__active_aiterator = self.__aiter__() + return await self.__active_aiterator.__anext__() + + async def _page_aiter(self, increment): + """Generator of pages of API responses. + + Args: + increment (bool): Flag indicating if the total number of results + should be incremented on each page. This is useful since a page + iterator will want to increment by results per page while an + items iterator will want to increment per item. + + Yields: + Page: each page of items from the API. + """ + page = await self._next_page() + while page is not None: + self.page_number += 1 + if increment: + self.num_results += page.num_items + yield page + page = await self._next_page() + + @abc.abstractmethod + async def _next_page(self): + """Get the next page in the iterator. + + This does nothing and is intended to be over-ridden by subclasses + to return the next :class:`Page`. + + Raises: + NotImplementedError: Always, this method is abstract. + """ + raise NotImplementedError + + +class AsyncGRPCIterator(AsyncIterator): + """A generic class for iterating through gRPC list responses. + + .. note:: The class does not take a ``page_token`` argument because it can + just be specified in the ``request``. + + Args: + client (google.cloud.client.Client): The API client. This unused by + this class, but kept to satisfy the :class:`Iterator` interface. + method (Callable[protobuf.Message]): A bound gRPC method that should + take a single message for the request. + request (protobuf.Message): The request message. + items_field (str): The field in the response message that has the + items for the page. + item_to_value (Callable[GRPCIterator, Any]): Callable to convert an + item from the type in the JSON response into a native object. Will + be called with the iterator and a single item. + request_token_field (str): The field in the request message used to + specify the page token. + response_token_field (str): The field in the response message that has + the token for the next page. + max_results (int): The maximum number of results to fetch. + + .. autoattribute:: pages + """ + + _DEFAULT_REQUEST_TOKEN_FIELD = "page_token" + _DEFAULT_RESPONSE_TOKEN_FIELD = "next_page_token" + + def __init__( + self, + client, + method, + request, + items_field, + item_to_value=_item_to_value_identity, + request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD, + response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD, + max_results=None, + ): + super().__init__(client, item_to_value, max_results=max_results) + self._method = method + self._request = request + self._items_field = items_field + self._request_token_field = request_token_field + self._response_token_field = response_token_field + + async def _next_page(self): + """Get the next page in the iterator. + + Returns: + Page: The next page in the iterator or :data:`None` if + there are no pages left. + """ + if not self._has_next_page(): + return None + + if self.next_page_token is not None: + setattr(self._request, self._request_token_field, self.next_page_token) + + response = await self._method(self._request) + + self.next_page_token = getattr(response, self._response_token_field) + items = getattr(response, self._items_field) + page = Page(self, items, self.item_to_value, raw_page=response) + + return page + + def _has_next_page(self): + """Determines whether or not there are more pages with results. + + Returns: + bool: Whether the iterator has more pages. + """ + if self.page_number == 0: + return True + + # Note: intentionally a falsy check instead of a None check. The RPC + # can return an empty string indicating no more pages. + if self.max_results is not None: + if self.num_results >= self.max_results: + return False + + return True if self.next_page_token else False diff --git a/py311/lib/python3.11/site-packages/google/api_core/path_template.py b/py311/lib/python3.11/site-packages/google/api_core/path_template.py new file mode 100644 index 0000000000000000000000000000000000000000..b8ebb2af92afd926750dfeac17209eea5bf80c74 --- /dev/null +++ b/py311/lib/python3.11/site-packages/google/api_core/path_template.py @@ -0,0 +1,346 @@ +# Copyright 2017 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expand and validate URL path templates. + +This module provides the :func:`expand` and :func:`validate` functions for +interacting with Google-style URL `path templates`_ which are commonly used +in Google APIs for `resource names`_. + +.. _path templates: https://github.com/googleapis/googleapis/blob + /57e2d376ac7ef48681554204a3ba78a414f2c533/google/api/http.proto#L212 +.. _resource names: https://cloud.google.com/apis/design/resource_names +""" + +from __future__ import unicode_literals + +from collections import deque +import copy +import functools +import re + +# Regular expression for extracting variable parts from a path template. +# The variables can be expressed as: +# +# - "*": a single-segment positional variable, for example: "books/*" +# - "**": a multi-segment positional variable, for example: "shelf/**/book/*" +# - "{name}": a single-segment wildcard named variable, for example +# "books/{name}" +# - "{name=*}: same as above. +# - "{name=**}": a multi-segment wildcard named variable, for example +# "shelf/{name=**}" +# - "{name=/path/*/**}": a multi-segment named variable with a sub-template. +_VARIABLE_RE = re.compile( + r""" + ( # Capture the entire variable expression + (?P\*\*?) # Match & capture * and ** positional variables. + | + # Match & capture named variables {name} + { + (?P[^/]+?) + # Optionally match and capture the named variable's template. + (?:=(?P