Add source batch 9/11

.gitattributes

@@ -246,3 +246,6 @@ source/ray/thirdparty_files/psutil/_psutil_linux.abi3.so filter=lfs diff=lfs mer
 source/regex/_regex.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
 source/rignore/rignore.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
 source/rpds/rpds.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+source/safetensors/_safetensors_rust.abi3.so filter=lfs diff=lfs merge=lfs -text
+source/sentencepiece/_sentencepiece.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+source/tiktoken/_tiktoken.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text

source/rpds_py-0.30.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/rpds_py-0.30.0.dist-info/METADATA

@@ -0,0 +1,99 @@
+Metadata-Version: 2.4
+Name: rpds-py
+Version: 0.30.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+License-File: LICENSE
+Summary: Python bindings to Rust's persistent data structures (rpds)
+Keywords: data structures,rust,persistent
+Author-email: Julian Berman <Julian+rpds@GrayVines.com>
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst; charset=UTF-8
+Project-URL: Documentation, https://rpds.readthedocs.io/
+Project-URL: Homepage, https://github.com/crate-py/rpds
+Project-URL: Issues, https://github.com/crate-py/rpds/issues/
+Project-URL: Funding, https://github.com/sponsors/Julian
+Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-rpds-py?utm_source=pypi-rpds-py&utm_medium=referral&utm_campaign=pypi-link
+Project-URL: Source, https://github.com/crate-py/rpds
+Project-URL: Upstream, https://github.com/orium/rpds
+
+===========
+``rpds.py``
+===========
+
+|PyPI| |Pythons| |CI|
+
+.. |PyPI| image:: https://img.shields.io/pypi/v/rpds-py.svg
+  :alt: PyPI version
+  :target: https://pypi.org/project/rpds-py/
+
+.. |Pythons| image:: https://img.shields.io/pypi/pyversions/rpds-py.svg
+  :alt: Supported Python versions
+  :target: https://pypi.org/project/rpds-py/
+
+.. |CI| image:: https://github.com/crate-py/rpds/workflows/CI/badge.svg
+  :alt: Build status
+  :target: https://github.com/crate-py/rpds/actions?query=workflow%3ACI
+
+.. |ReadTheDocs| image:: https://readthedocs.org/projects/referencing/badge/?version=stable&style=flat
+   :alt: ReadTheDocs status
+   :target: https://referencing.readthedocs.io/en/stable/
+
+
+Python bindings to the `Rust rpds crate <https://docs.rs/rpds/>`_ for persistent data structures.
+
+What's here is quite minimal (in transparency, it was written initially to support replacing ``pyrsistent`` in the `referencing library <https://github.com/python-jsonschema/referencing>`_).
+If you see something missing (which is very likely), a PR is definitely welcome to add it.
+
+Installation
+------------
+
+The distribution on PyPI is named ``rpds.py`` (equivalently ``rpds-py``), and thus can be installed via e.g.:
+
+.. code:: sh
+
+    $ pip install rpds-py
+
+Note that if you install ``rpds-py`` from source, you will need a Rust toolchain installed, as it is a build-time dependency.
+An example of how to do so in a ``Dockerfile`` can be found `here <https://github.com/bowtie-json-schema/bowtie/blob/e77fd93598cb6e7dc1b8b1f53c00e5aa410c201a/implementations/python-jsonschema/Dockerfile#L1-L8>`_.
+
+If you believe you are on a common platform which should have wheels built (i.e. and not need to compile from source), feel free to file an issue or pull request modifying the GitHub action used here to build wheels via ``maturin``.
+
+Usage
+-----
+
+Methods in general are named similarly to their ``rpds`` counterparts (rather than ``pyrsistent``\ 's conventions, though probably a full drop-in ``pyrsistent``\ -compatible wrapper module is a good addition at some point).
+
+.. code:: python
+
+    >>> from rpds import HashTrieMap, HashTrieSet, List
+
+    >>> m = HashTrieMap({"foo": "bar", "baz": "quux"})
+    >>> m.insert("spam", 37) == HashTrieMap({"foo": "bar", "baz": "quux", "spam": 37})
+    True
+    >>> m.remove("foo") == HashTrieMap({"baz": "quux"})
+    True
+
+    >>> s = HashTrieSet({"foo", "bar", "baz", "quux"})
+    >>> s.insert("spam") == HashTrieSet({"foo", "bar", "baz", "quux", "spam"})
+    True
+    >>> s.remove("foo") == HashTrieSet({"bar", "baz", "quux"})
+    True
+
+    >>> L = List([1, 3, 5])
+    >>> L.push_front(-1) == List([-1, 1, 3, 5])
+    True
+    >>> L.rest == List([3, 5])
+    True
+

source/rpds_py-0.30.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+rpds/__init__.py,sha256=w3MgXW7lpTCICw0KXbw20QX573_kbsEnWIeMsCAugvM,99
+rpds/__init__.pyi,sha256=am7x6oMa_pu_kv1NlolqJbPr6_UvCvoyxGKrDGSMKEk,2602
+rpds/__pycache__/__init__.cpython-312.pyc,,
+rpds/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rpds/rpds.cpython-312-x86_64-linux-gnu.so,sha256=YnSRYU7s221B-NVp0g5gVF1LPFzVnkzK3CdalA33MzY,1060936
+rpds_py-0.30.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+rpds_py-0.30.0.dist-info/METADATA,sha256=zAHQl925x-B6KclVPujdNc-SDoX7V097DCPWgTinFr8,4145
+rpds_py-0.30.0.dist-info/RECORD,,
+rpds_py-0.30.0.dist-info/WHEEL,sha256=m2ROzCpH5Kw6bN_3jKfw80jyQS9OqSulcWBhBkC07PU,147
+rpds_py-0.30.0.dist-info/licenses/LICENSE,sha256=MU5Okb47qpPA-0vMyeTpfNZD64ObBlr5IXgsIXX-mQk,1057

source/rpds_py-0.30.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.10.2)
+Root-Is-Purelib: false
+Tag: cp312-cp312-manylinux_2_17_x86_64
+Tag: cp312-cp312-manylinux2014_x86_64

source/rpds_py-0.30.0.dist-info/licenses/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Julian Berman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

source/safetensors-0.7.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/safetensors-0.7.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: safetensors
+Version: 0.7.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Dist: numpy>=1.21.6 ; extra == 'numpy'
+Requires-Dist: packaging ; extra == 'torch'
+Requires-Dist: safetensors[numpy] ; extra == 'torch'
+Requires-Dist: torch>=1.10 ; extra == 'torch'
+Requires-Dist: safetensors[numpy] ; extra == 'tensorflow'
+Requires-Dist: tensorflow>=2.11.0 ; extra == 'tensorflow'
+Requires-Dist: safetensors[numpy] ; extra == 'pinned-tf'
+Requires-Dist: tensorflow==2.18.0 ; extra == 'pinned-tf'
+Requires-Dist: safetensors[numpy] ; extra == 'jax'
+Requires-Dist: flax>=0.6.3 ; extra == 'jax'
+Requires-Dist: jax>=0.3.25 ; extra == 'jax'
+Requires-Dist: jaxlib>=0.3.25 ; extra == 'jax'
+Requires-Dist: mlx>=0.0.9 ; extra == 'mlx'
+Requires-Dist: safetensors[numpy] ; extra == 'paddlepaddle'
+Requires-Dist: paddlepaddle>=2.4.1 ; extra == 'paddlepaddle'
+Requires-Dist: ruff ; extra == 'quality'
+Requires-Dist: safetensors[numpy] ; extra == 'testing'
+Requires-Dist: h5py>=3.7.0 ; extra == 'testing'
+Requires-Dist: huggingface-hub>=0.12.1 ; extra == 'testing'
+Requires-Dist: setuptools-rust>=1.5.2 ; extra == 'testing'
+Requires-Dist: pytest>=7.2.0 ; extra == 'testing'
+Requires-Dist: pytest-benchmark>=4.0.0 ; extra == 'testing'
+Requires-Dist: hypothesis>=6.70.2 ; extra == 'testing'
+Requires-Dist: safetensors[numpy] ; extra == 'testingfree'
+Requires-Dist: huggingface-hub>=0.12.1 ; extra == 'testingfree'
+Requires-Dist: setuptools-rust>=1.5.2 ; extra == 'testingfree'
+Requires-Dist: pytest>=7.2.0 ; extra == 'testingfree'
+Requires-Dist: pytest-benchmark>=4.0.0 ; extra == 'testingfree'
+Requires-Dist: hypothesis>=6.70.2 ; extra == 'testingfree'
+Requires-Dist: safetensors[torch] ; extra == 'all'
+Requires-Dist: safetensors[numpy] ; extra == 'all'
+Requires-Dist: safetensors[pinned-tf] ; extra == 'all'
+Requires-Dist: safetensors[jax] ; extra == 'all'
+Requires-Dist: safetensors[paddlepaddle] ; extra == 'all'
+Requires-Dist: safetensors[quality] ; extra == 'all'
+Requires-Dist: safetensors[testing] ; extra == 'all'
+Requires-Dist: safetensors[all] ; extra == 'dev'
+Provides-Extra: numpy
+Provides-Extra: torch
+Provides-Extra: tensorflow
+Provides-Extra: pinned-tf
+Provides-Extra: jax
+Provides-Extra: mlx
+Provides-Extra: paddlepaddle
+Provides-Extra: quality
+Provides-Extra: testing
+Provides-Extra: testingfree
+Provides-Extra: all
+Provides-Extra: dev
+License-File: LICENSE
+Author-email: Nicolas Patry <patry.nicolas@protonmail.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/huggingface/safetensors
+Project-URL: Source, https://github.com/huggingface/safetensors
+
+## Installation
+
+```
+pip install safetensors
+```
+
+
+## Usage
+
+### Numpy
+
+```python
+from safetensors.numpy import save_file, load_file
+import numpy as np
+
+tensors = {
+   "a": np.zeros((2, 2)),
+   "b": np.zeros((2, 3), dtype=np.uint8)
+}
+
+save_file(tensors, "./model.safetensors")
+
+
+# Now loading
+loaded = load_file("./model.safetensors")
+```
+
+### Torch
+
+```python
+from safetensors.torch import save_file, load_file
+import torch
+
+tensors = {
+   "a": torch.zeros((2, 2)),
+   "b": torch.zeros((2, 3), dtype=torch.uint8)
+}
+
+save_file(tensors, "./model.safetensors")
+
+
+# Now loading
+loaded = load_file("./model.safetensors")
+```
+
+### Developing
+
+```
+# inside ./safetensors/bindings/python
+pip install .[dev]
+```
+Should be enough to install this library locally.
+
+### Testing
+
+```
+# inside ./safetensors/bindings/python
+pip install .[dev]
+pytest -sv tests/
+```
+

source/safetensors-0.7.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+safetensors-0.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+safetensors-0.7.0.dist-info/METADATA,sha256=aAkyb78XRE3VIKsZ8KVbehhHbb5SpDBGa79TiZZ8Kqo,4125
+safetensors-0.7.0.dist-info/RECORD,,
+safetensors-0.7.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+safetensors-0.7.0.dist-info/WHEEL,sha256=EsgGQg7OBGIn-zS1ipDRPjO8C2qSQ0GRrd2xuL_Pyq0,143
+safetensors-0.7.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+safetensors/__init__.py,sha256=wbzKZlAVgnAsjHmqryuSJCiADvpDZxNGCfj8VzY0At0,194
+safetensors/__init__.pyi,sha256=IpwsrzRWJA2yR8TxMEC3RHgM_5TiDgSFqyvrAxAa15U,4019
+safetensors/__pycache__/__init__.cpython-312.pyc,,
+safetensors/__pycache__/flax.cpython-312.pyc,,
+safetensors/__pycache__/mlx.cpython-312.pyc,,
+safetensors/__pycache__/numpy.cpython-312.pyc,,
+safetensors/__pycache__/paddle.cpython-312.pyc,,
+safetensors/__pycache__/tensorflow.cpython-312.pyc,,
+safetensors/__pycache__/torch.cpython-312.pyc,,
+safetensors/_safetensors_rust.abi3.so,sha256=h9qPRzPm7PhApdjisd6IjK0BLCN1URiRlWfsSAp9fgI,1216632
+safetensors/flax.py,sha256=T59elUqzVDyGYGdR78QzNEuwyAc8KrEO0EuLBSKOnUs,3853
+safetensors/mlx.py,sha256=IR51jRpcJq6epb0Agj8VsxI9xqBS6NjeAJnr-Ny0jJU,3850
+safetensors/numpy.py,sha256=rit_12-IfZtRgip_VLd8nPAcCXyeM2fPrCDZ7OiyxSY,5028
+safetensors/paddle.py,sha256=LrDwqQbwFnQXiY3M601IU7G6FBctX6tyHHK3_UH6lxE,8721
+safetensors/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+safetensors/tensorflow.py,sha256=AZ-O7-gM-JqTfjczZyCUAHm3Er-GSQnQWaFyY7mAIQc,3903
+safetensors/torch.py,sha256=U0acZVahLsxvqPa1GitMRiaByu6XVbmiBITGtuznBEY,18610

source/safetensors-0.7.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: maturin (1.10.2)
+Root-Is-Purelib: false
+Tag: cp38-abi3-manylinux_2_17_x86_64
+Tag: cp38-abi3-manylinux2014_x86_64

source/safetensors-0.7.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

source/safetensors/__init__.py

@@ -0,0 +1,10 @@
+# Re-export this
+from ._safetensors_rust import (  # noqa: F401
+    SafetensorError,
+    __version__,
+    deserialize,
+    safe_open,
+    _safe_open_handle,
+    serialize,
+    serialize_file,
+)

source/safetensors/__init__.pyi

@@ -0,0 +1,164 @@
+# Generated content DO NOT EDIT
+@staticmethod
+def deserialize(bytes):
+    """
+    Opens a safetensors lazily and returns tensors as asked
+
+    Args:
+        data (`bytes`):
+            The byte content of a file
+
+    Returns:
+        (`List[str, Dict[str, Dict[str, any]]]`):
+            The deserialized content is like:
+                [("tensor_name", {"shape": [2, 3], "dtype": "F32", "data": b"\0\0.." }), (...)]
+    """
+    pass
+
+@staticmethod
+def serialize(tensor_dict, metadata=None):
+    """
+    Serializes raw data.
+
+    Args:
+        tensor_dict (`Dict[str, Dict[Any]]`):
+            The tensor dict is like:
+                {"tensor_name": {"dtype": "F32", "shape": [2, 3], "data": b"\0\0"}}
+        metadata (`Dict[str, str]`, *optional*):
+            The optional purely text annotations
+
+    Returns:
+        (`bytes`):
+            The serialized content.
+    """
+    pass
+
+@staticmethod
+def serialize_file(tensor_dict, filename, metadata=None):
+    """
+    Serializes raw data into file.
+
+    Args:
+        tensor_dict (`Dict[str, Dict[Any]]`):
+            The tensor dict is like:
+                {"tensor_name": {"dtype": "F32", "shape": [2, 3], "data": b"\0\0"}}
+        filename (`str`, or `os.PathLike`):
+            The name of the file to write into.
+        metadata (`Dict[str, str]`, *optional*):
+            The optional purely text annotations
+
+    Returns:
+        (`NoneType`):
+            On success return None
+    """
+    pass
+
+class safe_open:
+    """
+    Opens a safetensors lazily and returns tensors as asked
+
+    Args:
+        filename (`str`, or `os.PathLike`):
+            The filename to open
+
+        framework (`str`):
+            The framework you want you tensors in. Supported values:
+            `pt`, `tf`, `flax`, `numpy`.
+
+        device (`str`, defaults to `"cpu"`):
+            The device on which you want the tensors.
+    """
+    def __init__(self, filename, framework, device=...):
+        pass
+
+    def __enter__(self):
+        """
+        Start the context manager
+        """
+        pass
+
+    def __exit__(self, _exc_type, _exc_value, _traceback):
+        """
+        Exits the context manager
+        """
+        pass
+
+    def get_slice(self, name):
+        """
+        Returns a full slice view object
+
+        Args:
+            name (`str`):
+                The name of the tensor you want
+
+        Returns:
+            (`PySafeSlice`):
+                A dummy object you can slice into to get a real tensor
+        Example:
+        ```python
+        from safetensors import safe_open
+
+        with safe_open("model.safetensors", framework="pt", device=0) as f:
+            tensor_part = f.get_slice("embedding")[:, ::8]
+
+        ```
+        """
+        pass
+
+    def get_tensor(self, name):
+        """
+        Returns a full tensor
+
+        Args:
+            name (`str`):
+                The name of the tensor you want
+
+        Returns:
+            (`Tensor`):
+                The tensor in the framework you opened the file for.
+
+        Example:
+        ```python
+        from safetensors import safe_open
+
+        with safe_open("model.safetensors", framework="pt", device=0) as f:
+            tensor = f.get_tensor("embedding")
+
+        ```
+        """
+        pass
+
+    def keys(self):
+        """
+        Returns the names of the tensors in the file.
+
+        Returns:
+            (`List[str]`):
+                The name of the tensors contained in that file
+        """
+        pass
+
+    def metadata(self):
+        """
+        Return the special non tensor information in the header
+
+        Returns:
+            (`Dict[str, str]`):
+                The freeform metadata.
+        """
+        pass
+
+    def offset_keys(self):
+        """
+        Returns the names of the tensors in the file, ordered by offset.
+
+        Returns:
+            (`List[str]`):
+                The name of the tensors contained in that file
+        """
+        pass
+
+class SafetensorError(Exception):
+    """
+    Custom Python Exception for Safetensor errors.
+    """

source/safetensors/_safetensors_rust.abi3.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87da8f4733e6ecf840a5d8e2b1de888cad012c23755118919567ec480a7d7e02
+size 1216632

source/safetensors/flax.py

@@ -0,0 +1,138 @@
+import os
+from typing import Dict, Optional, Union
+
+import numpy as np
+
+import jax.numpy as jnp
+from jax import Array
+from safetensors import numpy, safe_open
+
+
+def save(tensors: Dict[str, Array], metadata: Optional[Dict[str, str]] = None) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, Array]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.flax import save
+    from jax import numpy as jnp
+
+    tensors = {"embedding": jnp.zeros((512, 1024)), "attention": jnp.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    np_tensors = _jnp2np(tensors)
+    return numpy.save(np_tensors, metadata=metadata)
+
+
+def save_file(
+    tensors: Dict[str, Array],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+) -> None:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, Array]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.flax import save_file
+    from jax import numpy as jnp
+
+    tensors = {"embedding": jnp.zeros((512, 1024)), "attention": jnp.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    np_tensors = _jnp2np(tensors)
+    return numpy.save_file(np_tensors, filename, metadata=metadata)
+
+
+def load(data: bytes) -> Dict[str, Array]:
+    """
+    Loads a safetensors file into flax format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, Array]`: dictionary that contains name as key, value as `Array` on cpu
+
+    Example:
+
+    ```python
+    from safetensors.flax import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    flat = numpy.load(data)
+    return _np2jnp(flat)
+
+
+def load_file(filename: Union[str, os.PathLike]) -> Dict[str, Array]:
+    """
+    Loads a safetensors file into flax format.
+
+    Args:
+        filename (`str`, or `os.PathLike`)):
+            The name of the file which contains the tensors
+
+    Returns:
+        `Dict[str, Array]`: dictionary that contains name as key, value as `Array`
+
+    Example:
+
+    ```python
+    from safetensors.flax import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    with safe_open(filename, framework="flax") as f:
+        for k in f.offset_keys():
+            result[k] = f.get_tensor(k)
+    return result
+
+
+def _np2jnp(numpy_dict: Dict[str, np.ndarray]) -> Dict[str, Array]:
+    for k, v in numpy_dict.items():
+        numpy_dict[k] = jnp.array(v)
+    return numpy_dict
+
+
+def _jnp2np(jnp_dict: Dict[str, Array]) -> Dict[str, np.array]:
+    for k, v in jnp_dict.items():
+        jnp_dict[k] = np.asarray(v)
+    return jnp_dict

source/safetensors/mlx.py

@@ -0,0 +1,140 @@
+import os
+from typing import Dict, Optional, Union
+
+import numpy as np
+
+import mlx.core as mx
+from safetensors import numpy, safe_open
+
+
+def save(
+    tensors: Dict[str, mx.array], metadata: Optional[Dict[str, str]] = None
+) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, mx.array]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.mlx import save
+    import mlx.core as mx
+
+    tensors = {"embedding": mx.zeros((512, 1024)), "attention": mx.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    np_tensors = _mx2np(tensors)
+    return numpy.save(np_tensors, metadata=metadata)
+
+
+def save_file(
+    tensors: Dict[str, mx.array],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+) -> None:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, mx.array]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.mlx import save_file
+    import mlx.core as mx
+
+    tensors = {"embedding": mx.zeros((512, 1024)), "attention": mx.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    np_tensors = _mx2np(tensors)
+    return numpy.save_file(np_tensors, filename, metadata=metadata)
+
+
+def load(data: bytes) -> Dict[str, mx.array]:
+    """
+    Loads a safetensors file into MLX format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, mx.array]`: dictionary that contains name as key, value as `mx.array`
+
+    Example:
+
+    ```python
+    from safetensors.mlx import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    flat = numpy.load(data)
+    return _np2mx(flat)
+
+
+def load_file(filename: Union[str, os.PathLike]) -> Dict[str, mx.array]:
+    """
+    Loads a safetensors file into MLX format.
+
+    Args:
+        filename (`str`, or `os.PathLike`)):
+            The name of the file which contains the tensors
+
+    Returns:
+        `Dict[str, mx.array]`: dictionary that contains name as key, value as `mx.array`
+
+    Example:
+
+    ```python
+    from safetensors.flax import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    with safe_open(filename, framework="mlx") as f:
+        for k in f.offset_keys():
+            result[k] = f.get_tensor(k)
+    return result
+
+
+def _np2mx(numpy_dict: Dict[str, np.ndarray]) -> Dict[str, mx.array]:
+    for k, v in numpy_dict.items():
+        numpy_dict[k] = mx.array(v)
+    return numpy_dict
+
+
+def _mx2np(mx_dict: Dict[str, mx.array]) -> Dict[str, np.array]:
+    new_dict = {}
+    for k, v in mx_dict.items():
+        new_dict[k] = np.asarray(v)
+    return new_dict

source/safetensors/numpy.py

@@ -0,0 +1,187 @@
+import os
+import sys
+from typing import Dict, Optional, Union
+
+import numpy as np
+
+from safetensors import deserialize, safe_open, serialize, serialize_file
+
+
+def _tobytes(tensor: np.ndarray) -> bytes:
+    if not _is_little_endian(tensor):
+        tensor = tensor.byteswap(inplace=False)
+    return tensor.tobytes()
+
+
+def save(
+    tensor_dict: Dict[str, np.ndarray], metadata: Optional[Dict[str, str]] = None
+) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensor_dict (`Dict[str, np.ndarray]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.numpy import save
+    import numpy as np
+
+    tensors = {"embedding": np.zeros((512, 1024)), "attention": np.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    flattened = {
+        k: {"dtype": v.dtype.name, "shape": v.shape, "data": _tobytes(v)}
+        for k, v in tensor_dict.items()
+    }
+    serialized = serialize(flattened, metadata=metadata)
+    result = bytes(serialized)
+    return result
+
+
+def save_file(
+    tensor_dict: Dict[str, np.ndarray],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+) -> None:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensor_dict (`Dict[str, np.ndarray]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.numpy import save_file
+    import numpy as np
+
+    tensors = {"embedding": np.zeros((512, 1024)), "attention": np.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    flattened = {
+        k: {"dtype": v.dtype.name, "shape": v.shape, "data": _tobytes(v)}
+        for k, v in tensor_dict.items()
+    }
+    serialize_file(flattened, filename, metadata=metadata)
+
+
+def load(data: bytes) -> Dict[str, np.ndarray]:
+    """
+    Loads a safetensors file into numpy format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, np.ndarray]`: dictionary that contains name as key, value as `np.ndarray` on cpu
+
+    Example:
+
+    ```python
+    from safetensors.numpy import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    flat = deserialize(data)
+    return _view2np(flat)
+
+
+def load_file(filename: Union[str, os.PathLike]) -> Dict[str, np.ndarray]:
+    """
+    Loads a safetensors file into numpy format.
+
+    Args:
+        filename (`str`, or `os.PathLike`)):
+            The name of the file which contains the tensors
+
+    Returns:
+        `Dict[str, np.ndarray]`: dictionary that contains name as key, value as `np.ndarray`
+
+    Example:
+
+    ```python
+    from safetensors.numpy import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    with safe_open(filename, framework="np") as f:
+        for k in f.offset_keys():
+            result[k] = f.get_tensor(k)
+    return result
+
+
+_TYPES = {
+    "F64": np.float64,
+    "F32": np.float32,
+    "F16": np.float16,
+    "I64": np.int64,
+    "U64": np.uint64,
+    "I32": np.int32,
+    "U32": np.uint32,
+    "I16": np.int16,
+    "U16": np.uint16,
+    "I8": np.int8,
+    "U8": np.uint8,
+    "BOOL": bool,
+    "C64": np.complex64,
+}
+
+
+def _getdtype(dtype_str: str) -> np.dtype:
+    return _TYPES[dtype_str]
+
+
+def _view2np(safeview) -> Dict[str, np.ndarray]:
+    result = {}
+    for k, v in safeview:
+        dtype = _getdtype(v["dtype"])
+        arr = np.frombuffer(v["data"], dtype=dtype).reshape(v["shape"])
+        result[k] = arr
+    return result
+
+
+def _is_little_endian(tensor: np.ndarray) -> bool:
+    byteorder = tensor.dtype.byteorder
+    if byteorder == "=":
+        if sys.byteorder == "little":
+            return True
+        else:
+            return False
+    elif byteorder == "|":
+        return True
+    elif byteorder == "<":
+        return True
+    elif byteorder == ">":
+        return False
+    raise ValueError(f"Unexpected byte order {byteorder}")

source/safetensors/paddle.py

@@ -0,0 +1,290 @@
+import os
+import sys
+from typing import Any, Dict, Optional, Union
+
+import numpy as np
+import paddle
+
+from safetensors import numpy, deserialize, safe_open, serialize, serialize_file
+
+
+def save(
+    tensors: Dict[str, paddle.Tensor], metadata: Optional[Dict[str, str]] = None
+) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, paddle.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.paddle import save
+    import paddle
+
+    tensors = {"embedding": paddle.zeros((512, 1024)), "attention": paddle.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    serialized = serialize(_flatten(tensors), metadata=metadata)
+    result = bytes(serialized)
+    return result
+
+
+def save_file(
+    tensors: Dict[str, paddle.Tensor],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+) -> None:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, paddle.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.paddle import save_file
+    import paddle
+
+    tensors = {"embedding": paddle.zeros((512, 1024)), "attention": paddle.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    serialize_file(_flatten(tensors), filename, metadata=metadata)
+
+
+def load(data: bytes, device: str = "cpu") -> Dict[str, paddle.Tensor]:
+    """
+    Loads a safetensors file into paddle format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, paddle.Tensor]`: dictionary that contains name as key, value as `paddle.Tensor` on cpu
+
+    Example:
+
+    ```python
+    from safetensors.paddle import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    if paddle.__version__ >= "3.2.0":
+        flat = deserialize(data)
+        return _view2paddle(flat, device)
+    else:
+        flat = numpy.load(data)
+        return _np2paddle(flat, device)
+
+
+def load_file(
+    filename: Union[str, os.PathLike], device="cpu"
+) -> Dict[str, paddle.Tensor]:
+    """
+    Loads a safetensors file into paddle format.
+
+    Args:
+        filename (`str`, or `os.PathLike`)):
+            The name of the file which contains the tensors
+        device (`Union[Dict[str, any], str]`, *optional*, defaults to `cpu`):
+            The device where the tensors need to be located after load.
+            available options are all regular paddle device locations
+
+    Returns:
+        `Dict[str, paddle.Tensor]`: dictionary that contains name as key, value as `paddle.Tensor`
+
+    Example:
+
+    ```python
+    from safetensors.paddle import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    if paddle.__version__ >= "3.2.0":
+        with safe_open(filename, framework="paddle", device=device) as f:
+            for k in f.offset_keys():
+                result[k] = f.get_tensor(k)
+    else:
+        flat = numpy.load_file(filename)
+        result = _np2paddle(flat, device)
+    return result
+
+
+def _np2paddle(
+    numpy_dict: Dict[str, np.ndarray], device: str = "cpu"
+) -> Dict[str, paddle.Tensor]:
+    for k, v in numpy_dict.items():
+        numpy_dict[k] = paddle.to_tensor(v, place=device)
+    return numpy_dict
+
+
+def _paddle2np(paddle_dict: Dict[str, paddle.Tensor]) -> Dict[str, np.array]:
+    for k, v in paddle_dict.items():
+        paddle_dict[k] = v.detach().cpu().numpy()
+    return paddle_dict
+
+
+_SIZE = {
+    paddle.int64: 8,
+    paddle.float32: 4,
+    paddle.int32: 4,
+    paddle.bfloat16: 2,
+    paddle.float16: 2,
+    paddle.int16: 2,
+    paddle.uint8: 1,
+    paddle.int8: 1,
+    paddle.bool: 1,
+    paddle.float64: 8,
+    paddle.float8_e4m3fn: 1,
+    paddle.float8_e5m2: 1,
+    paddle.complex64: 8,
+    # XXX: These are not supported yet in paddle
+    # paddle.uint64: 8,
+    # paddle.uint32: 4,
+    # paddle.uint16: 2,
+    # paddle.float8_e8m0: 1,
+    # paddle.float4_e2m1_x2: 1,
+}
+
+_TYPES = {
+    "F64": paddle.float64,
+    "F32": paddle.float32,
+    "F16": paddle.float16,
+    "BF16": paddle.bfloat16,
+    "I64": paddle.int64,
+    "I32": paddle.int32,
+    "I16": paddle.int16,
+    "I8": paddle.int8,
+    "U8": paddle.uint8,
+    "BOOL": paddle.bool,
+    "F8_E4M3": paddle.float8_e4m3fn,
+    "F8_E5M2": paddle.float8_e5m2,
+}
+
+NPDTYPES = {
+    paddle.int64: np.int64,
+    paddle.float32: np.float32,
+    paddle.int32: np.int32,
+    # XXX: This is ok because both have the same width
+    paddle.bfloat16: np.float16,
+    paddle.float16: np.float16,
+    paddle.int16: np.int16,
+    paddle.uint8: np.uint8,
+    paddle.int8: np.int8,
+    paddle.bool: bool,
+    paddle.float64: np.float64,
+    # XXX: This is ok because both have the same width and byteswap is a no-op anyway
+    paddle.float8_e4m3fn: np.uint8,
+    paddle.float8_e5m2: np.uint8,
+}
+
+
+def _getdtype(dtype_str: str) -> paddle.dtype:
+    return _TYPES[dtype_str]
+
+
+def _view2paddle(safeview, device) -> Dict[str, paddle.Tensor]:
+    result = {}
+    for k, v in safeview:
+        dtype = _getdtype(v["dtype"])
+        if len(v["data"]) == 0:
+            # Workaround because frombuffer doesn't accept zero-size tensors
+            assert any(x == 0 for x in v["shape"])
+            arr = paddle.empty(v["shape"], dtype=dtype)
+        else:
+            arr = paddle.base.core.frombuffer(v["data"], dtype).reshape(v["shape"])
+            if device != "cpu":
+                arr = arr.to(device)
+        if sys.byteorder == "big":
+            arr = paddle.to_tensor(arr.numpy().byteswap(inplace=False), place=device)
+        result[k] = arr
+
+    return result
+
+
+def _tobytes(tensor: paddle.Tensor, name: str) -> bytes:
+    if not tensor.is_contiguous():
+        raise ValueError(
+            f"You are trying to save a non contiguous tensor: `{name}` which is not allowed. It either means you"
+            " are trying to save tensors which are reference of each other in which case it's recommended to save"
+            " only the full tensors, and reslice at load time, or simply call `.contiguous()` on your tensor to"
+            " pack it before saving."
+        )
+    if not tensor.place.is_cpu_place():
+        # Moving tensor to cpu before saving
+        tensor = tensor.cpu()
+
+    import ctypes
+
+    import numpy as np
+
+    # When shape is empty (scalar), np.prod returns a float
+    # we need a int for the following calculations
+    length = int(np.prod(tensor.shape).item())
+    bytes_per_item = _SIZE[tensor.dtype]
+
+    total_bytes = length * bytes_per_item
+
+    ptr = tensor.data_ptr()
+    if ptr == 0:
+        return b""
+    newptr = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_ubyte))
+    data = np.ctypeslib.as_array(newptr, (total_bytes,))  # no internal copy
+    if sys.byteorder == "big":
+        npdtype = NPDTYPES[tensor.dtype]
+        # Not in place as that would potentially modify a live running model
+        data = data.view(npdtype).byteswap(inplace=False)
+    return data.tobytes()
+
+
+def _flatten(tensors: Dict[str, paddle.Tensor]) -> Dict[str, Dict[str, Any]]:
+    if not isinstance(tensors, dict):
+        raise ValueError(
+            f"Expected a dict of [str, paddle.Tensor] but received {type(tensors)}"
+        )
+
+    for k, v in tensors.items():
+        if not isinstance(v, paddle.Tensor):
+            raise ValueError(
+                f"Key `{k}` is invalid, expected paddle.Tensor but received {type(v)}"
+            )
+
+    return {
+        k: {
+            "dtype": str(v.dtype).split(".")[-1],
+            "shape": v.shape,
+            "data": _tobytes(v, k),
+        }
+        for k, v in tensors.items()
+    }

source/safetensors/tensorflow.py

@@ -0,0 +1,139 @@
+import os
+from typing import Dict, Optional, Union
+
+import numpy as np
+import tensorflow as tf
+
+from safetensors import numpy, safe_open
+
+
+def save(
+    tensors: Dict[str, tf.Tensor], metadata: Optional[Dict[str, str]] = None
+) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, tf.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.tensorflow import save
+    import tensorflow as tf
+
+    tensors = {"embedding": tf.zeros((512, 1024)), "attention": tf.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    np_tensors = _tf2np(tensors)
+    return numpy.save(np_tensors, metadata=metadata)
+
+
+def save_file(
+    tensors: Dict[str, tf.Tensor],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+) -> None:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, tf.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.tensorflow import save_file
+    import tensorflow as tf
+
+    tensors = {"embedding": tf.zeros((512, 1024)), "attention": tf.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    np_tensors = _tf2np(tensors)
+    return numpy.save_file(np_tensors, filename, metadata=metadata)
+
+
+def load(data: bytes) -> Dict[str, tf.Tensor]:
+    """
+    Loads a safetensors file into tensorflow format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, tf.Tensor]`: dictionary that contains name as key, value as `tf.Tensor` on cpu
+
+    Example:
+
+    ```python
+    from safetensors.tensorflow import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    flat = numpy.load(data)
+    return _np2tf(flat)
+
+
+def load_file(filename: Union[str, os.PathLike]) -> Dict[str, tf.Tensor]:
+    """
+    Loads a safetensors file into tensorflow format.
+
+    Args:
+        filename (`str`, or `os.PathLike`)):
+            The name of the file which contains the tensors
+
+    Returns:
+        `Dict[str, tf.Tensor]`: dictionary that contains name as key, value as `tf.Tensor`
+
+    Example:
+
+    ```python
+    from safetensors.tensorflow import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    with safe_open(filename, framework="tf") as f:
+        for k in f.offset_keys():
+            result[k] = f.get_tensor(k)
+    return result
+
+
+def _np2tf(numpy_dict: Dict[str, np.ndarray]) -> Dict[str, tf.Tensor]:
+    for k, v in numpy_dict.items():
+        numpy_dict[k] = tf.convert_to_tensor(v)
+    return numpy_dict
+
+
+def _tf2np(tf_dict: Dict[str, tf.Tensor]) -> Dict[str, np.array]:
+    for k, v in tf_dict.items():
+        tf_dict[k] = v.numpy()
+    return tf_dict

source/safetensors/torch.py

@@ -0,0 +1,550 @@
+import os
+import sys
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from packaging.version import Version
+
+import torch
+
+from safetensors import deserialize, safe_open, serialize, serialize_file
+
+
+def storage_ptr(tensor: torch.Tensor) -> int:
+    try:
+        return tensor.untyped_storage().data_ptr()
+    except Exception:
+        # Fallback for torch==1.10
+        try:
+            return tensor.storage().data_ptr()
+        except NotImplementedError:
+            # Fallback for meta storage
+            return 0
+
+
+def _end_ptr(tensor: torch.Tensor) -> int:
+    if tensor.nelement():
+        stop = tensor.view(-1)[-1].data_ptr() + _SIZE[tensor.dtype]
+    else:
+        stop = tensor.data_ptr()
+    return stop
+
+
+def storage_size(tensor: torch.Tensor) -> int:
+    try:
+        return tensor.untyped_storage().nbytes()
+    except AttributeError:
+        # Fallback for torch==1.10
+        try:
+            return tensor.storage().size() * _SIZE[tensor.dtype]
+        except NotImplementedError:
+            # Fallback for meta storage
+            # On torch >=2.0 this is the tensor size
+            return tensor.nelement() * _SIZE[tensor.dtype]
+
+
+def _filter_shared_not_shared(
+    tensors: List[Set[str]], state_dict: Dict[str, torch.Tensor]
+) -> List[Set[str]]:
+    filtered_tensors = []
+    for shared in tensors:
+        if len(shared) < 2:
+            filtered_tensors.append(shared)
+            continue
+
+        areas = []
+        for name in shared:
+            tensor = state_dict[name]
+            areas.append((tensor.data_ptr(), _end_ptr(tensor), name))
+        areas.sort()
+
+        _, last_stop, last_name = areas[0]
+        filtered_tensors.append({last_name})
+        for start, stop, name in areas[1:]:
+            if start >= last_stop:
+                filtered_tensors.append({name})
+            else:
+                filtered_tensors[-1].add(name)
+            last_stop = stop
+
+    return filtered_tensors
+
+
+def _find_shared_tensors(state_dict: Dict[str, torch.Tensor]) -> List[Set[str]]:
+    tensors = defaultdict(set)
+    for k, v in state_dict.items():
+        if (
+            v.device != torch.device("meta")
+            and storage_ptr(v) != 0
+            and storage_size(v) != 0
+        ):
+            # Need to add device as key because of multiple GPU.
+            tensors[(v.device, storage_ptr(v), storage_size(v))].add(k)
+    tensors = list(sorted(tensors.values()))
+    tensors = _filter_shared_not_shared(tensors, state_dict)
+    return tensors
+
+
+def _is_complete(tensor: torch.Tensor) -> bool:
+    return tensor.data_ptr() == storage_ptr(tensor) and tensor.nelement() * _SIZE[
+        tensor.dtype
+    ] == storage_size(tensor)
+
+
+def _remove_duplicate_names(
+    state_dict: Dict[str, torch.Tensor],
+    *,
+    preferred_names: Optional[List[str]] = None,
+    discard_names: Optional[List[str]] = None,
+) -> Dict[str, List[str]]:
+    if preferred_names is None:
+        preferred_names = []
+    preferred_names = set(preferred_names)
+    if discard_names is None:
+        discard_names = []
+    discard_names = set(discard_names)
+
+    shareds = _find_shared_tensors(state_dict)
+    to_remove = defaultdict(list)
+    for shared in shareds:
+        complete_names = set(
+            [name for name in shared if _is_complete(state_dict[name])]
+        )
+        if not complete_names:
+            raise RuntimeError(
+                "Error while trying to find names to remove to save state dict, but found no suitable name to keep"
+                f" for saving amongst: {shared}. None is covering the entire storage.Refusing to save/load the model"
+                " since you could be storing much more memory than needed. Please refer to"
+                " https://huggingface.co/docs/safetensors/torch_shared_tensors for more information. Or open an"
+                " issue."
+            )
+
+        keep_name = sorted(list(complete_names))[0]
+
+        # Mechanism to preferentially select keys to keep
+        # coming from the on-disk file to allow
+        # loading models saved with a different choice
+        # of keep_name
+        preferred = complete_names.difference(discard_names)
+        if preferred:
+            keep_name = sorted(list(preferred))[0]
+
+        if preferred_names:
+            preferred = preferred_names.intersection(complete_names)
+            if preferred:
+                keep_name = sorted(list(preferred))[0]
+        for name in sorted(shared):
+            if name != keep_name:
+                to_remove[keep_name].append(name)
+    return to_remove
+
+
+def save_model(
+    model: torch.nn.Module,
+    filename: str,
+    metadata: Optional[Dict[str, str]] = None,
+    force_contiguous: bool = True,
+):
+    """
+    Saves a given torch model to specified filename.
+    This method exists specifically to avoid tensor sharing issues which are
+    not allowed in `safetensors`. [More information on tensor sharing](../torch_shared_tensors)
+
+    Args:
+        model (`torch.nn.Module`):
+            The model to save on disk.
+        filename (`str`):
+            The filename location to save the file
+        metadata (`Dict[str, str]`, *optional*):
+            Extra information to save along with the file.
+            Some metadata will be added for each dropped tensors.
+            This information will not be enough to recover the entire
+            shared structure but might help understanding things
+        force_contiguous (`boolean`, *optional*, defaults to True):
+            Forcing the state_dict to be saved as contiguous tensors.
+            This has no effect on the correctness of the model, but it
+            could potentially change performance if the layout of the tensor
+            was chosen specifically for that reason.
+    """
+    state_dict = model.state_dict()
+    to_removes = _remove_duplicate_names(state_dict)
+
+    for kept_name, to_remove_group in to_removes.items():
+        for to_remove in to_remove_group:
+            if metadata is None:
+                metadata = {}
+
+            if to_remove not in metadata:
+                # Do not override user data
+                metadata[to_remove] = kept_name
+            del state_dict[to_remove]
+    if force_contiguous:
+        state_dict = {k: v.contiguous() for k, v in state_dict.items()}
+    try:
+        save_file(state_dict, filename, metadata=metadata)
+    except ValueError as e:
+        msg = str(e)
+        msg += " Or use save_model(..., force_contiguous=True), read the docs for potential caveats."
+        raise ValueError(msg)
+
+
+def load_model(
+    model: torch.nn.Module,
+    filename: Union[str, os.PathLike],
+    strict: bool = True,
+    device: Union[str, int] = "cpu",
+) -> Tuple[List[str], List[str]]:
+    """
+    Loads a given filename onto a torch model.
+    This method exists specifically to avoid tensor sharing issues which are
+    not allowed in `safetensors`. [More information on tensor sharing](../torch_shared_tensors)
+
+    Args:
+        model (`torch.nn.Module`):
+            The model to load onto.
+        filename (`str`, or `os.PathLike`):
+            The filename location to load the file from.
+        strict (`bool`, *optional*, defaults to True):
+            Whether to fail if you're missing keys or having unexpected ones.
+            When false, the function simply returns missing and unexpected names.
+        device (`Union[str, int]`, *optional*, defaults to `cpu`):
+            The device where the tensors need to be located after load.
+            available options are all regular torch device locations.
+
+    Returns:
+        `(missing, unexpected): (List[str], List[str])`
+            `missing` are names in the model which were not modified during loading
+            `unexpected` are names that are on the file, but weren't used during
+            the load.
+    """
+    state_dict = load_file(filename, device=device)
+    model_state_dict = model.state_dict()
+    to_removes = _remove_duplicate_names(
+        model_state_dict, preferred_names=state_dict.keys()
+    )
+    missing, unexpected = model.load_state_dict(state_dict, strict=False)
+    missing = set(missing)
+    for to_remove_group in to_removes.values():
+        for to_remove in to_remove_group:
+            if to_remove not in missing:
+                unexpected.append(to_remove)
+            else:
+                missing.remove(to_remove)
+    if strict and (missing or unexpected):
+        missing_keys = ", ".join([f'"{k}"' for k in sorted(missing)])
+        unexpected_keys = ", ".join([f'"{k}"' for k in sorted(unexpected)])
+        error = f"Error(s) in loading state_dict for {model.__class__.__name__}:"
+        if missing:
+            error += f"\n    Missing key(s) in state_dict: {missing_keys}"
+        if unexpected:
+            error += f"\n    Unexpected key(s) in state_dict: {unexpected_keys}"
+        raise RuntimeError(error)
+    return missing, unexpected
+
+
+def save(
+    tensors: Dict[str, torch.Tensor], metadata: Optional[Dict[str, str]] = None
+) -> bytes:
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, torch.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `bytes`: The raw bytes representing the format
+
+    Example:
+
+    ```python
+    from safetensors.torch import save
+    import torch
+
+    tensors = {"embedding": torch.zeros((512, 1024)), "attention": torch.zeros((256, 256))}
+    byte_data = save(tensors)
+    ```
+    """
+    serialized = serialize(_flatten(tensors), metadata=metadata)
+    result = bytes(serialized)
+    return result
+
+
+def save_file(
+    tensors: Dict[str, torch.Tensor],
+    filename: Union[str, os.PathLike],
+    metadata: Optional[Dict[str, str]] = None,
+):
+    """
+    Saves a dictionary of tensors into raw bytes in safetensors format.
+
+    Args:
+        tensors (`Dict[str, torch.Tensor]`):
+            The incoming tensors. Tensors need to be contiguous and dense.
+        filename (`str`, or `os.PathLike`)):
+            The filename we're saving into.
+        metadata (`Dict[str, str]`, *optional*, defaults to `None`):
+            Optional text only metadata you might want to save in your header.
+            For instance it can be useful to specify more about the underlying
+            tensors. This is purely informative and does not affect tensor loading.
+
+    Returns:
+        `None`
+
+    Example:
+
+    ```python
+    from safetensors.torch import save_file
+    import torch
+
+    tensors = {"embedding": torch.zeros((512, 1024)), "attention": torch.zeros((256, 256))}
+    save_file(tensors, "model.safetensors")
+    ```
+    """
+    serialize_file(_flatten(tensors), filename, metadata=metadata)
+
+
+def load_file(
+    filename: Union[str, os.PathLike], device: Union[str, int] = "cpu"
+) -> Dict[str, torch.Tensor]:
+    """
+    Loads a safetensors file into torch format.
+
+    Args:
+        filename (`str`, or `os.PathLike`):
+            The name of the file which contains the tensors
+        device (`Union[str, int]`, *optional*, defaults to `cpu`):
+            The device where the tensors need to be located after load.
+            available options are all regular torch device locations.
+
+    Returns:
+        `Dict[str, torch.Tensor]`: dictionary that contains name as key, value as `torch.Tensor`
+
+    Example:
+
+    ```python
+    from safetensors.torch import load_file
+
+    file_path = "./my_folder/bert.safetensors"
+    loaded = load_file(file_path)
+    ```
+    """
+    result = {}
+    with safe_open(filename, framework="pt", device=device) as f:
+        for k in f.offset_keys():
+            result[k] = f.get_tensor(k)
+    return result
+
+
+def load(data: bytes) -> Dict[str, torch.Tensor]:
+    """
+    Loads a safetensors file into torch format from pure bytes.
+
+    Args:
+        data (`bytes`):
+            The content of a safetensors file
+
+    Returns:
+        `Dict[str, torch.Tensor]`: dictionary that contains name as key, value as `torch.Tensor` on cpu
+
+    Example:
+
+    ```python
+    from safetensors.torch import load
+
+    file_path = "./my_folder/bert.safetensors"
+    with open(file_path, "rb") as f:
+        data = f.read()
+
+    loaded = load(data)
+    ```
+    """
+    flat = deserialize(data)
+    return _view2torch(flat)
+
+
+# torch.float8 formats require 2.1; we do not support these dtypes on earlier versions
+_float8_e4m3fn = getattr(torch, "float8_e4m3fn", None)
+_float8_e5m2 = getattr(torch, "float8_e5m2", None)
+_float8_e8m0 = getattr(torch, "float8_e8m0fnu", None)
+_float4_e2m1_x2 = getattr(torch, "float4_e2m1fn_x2", None)
+
+_SIZE = {
+    torch.int64: 8,
+    torch.float32: 4,
+    torch.int32: 4,
+    torch.bfloat16: 2,
+    torch.float16: 2,
+    torch.int16: 2,
+    torch.uint8: 1,
+    torch.int8: 1,
+    torch.bool: 1,
+    torch.float64: 8,
+    torch.complex64: 8,
+    _float8_e4m3fn: 1,
+    _float8_e5m2: 1,
+    _float8_e8m0: 1,
+    _float4_e2m1_x2: 1,
+}
+if Version(torch.__version__) >= Version("2.3.0"):
+    _SIZE.update(
+        {
+            torch.uint64: 8,
+            torch.uint32: 4,
+            torch.uint16: 2,
+        }
+    )
+
+_TYPES = {
+    "F64": torch.float64,
+    "F32": torch.float32,
+    "F16": torch.float16,
+    "BF16": torch.bfloat16,
+    "I64": torch.int64,
+    "I32": torch.int32,
+    "I16": torch.int16,
+    "I8": torch.int8,
+    "U8": torch.uint8,
+    "BOOL": torch.bool,
+    "F8_E4M3": _float8_e4m3fn,
+    "F8_E5M2": _float8_e5m2,
+    "C64": torch.complex64,
+}
+if Version(torch.__version__) >= Version("2.3.0"):
+    _TYPES.update(
+        {
+            "U64": torch.uint64,
+            "U32": torch.uint32,
+            "U16": torch.uint16,
+        }
+    )
+
+
+def _getdtype(dtype_str: str) -> torch.dtype:
+    return _TYPES[dtype_str]
+
+
+def _view2torch(safeview) -> Dict[str, torch.Tensor]:
+    result = {}
+    for k, v in safeview:
+        dtype = _getdtype(v["dtype"])
+        if len(v["data"]) == 0:
+            # Workaround because frombuffer doesn't accept zero-size tensors
+            assert any(x == 0 for x in v["shape"])
+            arr = torch.empty(v["shape"], dtype=dtype)
+        else:
+            arr = torch.frombuffer(v["data"], dtype=dtype).reshape(v["shape"])
+        if sys.byteorder == "big":
+            arr = torch.from_numpy(arr.numpy().byteswap(inplace=False))
+        result[k] = arr
+
+    return result
+
+
+def _tobytes(tensor: torch.Tensor, name: str) -> bytes:
+    if tensor.layout != torch.strided:
+        raise ValueError(
+            f"You are trying to save a sparse tensor: `{name}` which this library does not support."
+            " You can make it a dense tensor before saving with `.to_dense()` but be aware this might"
+            " make a much larger file than needed."
+        )
+
+    if not tensor.is_contiguous():
+        raise ValueError(
+            f"You are trying to save a non contiguous tensor: `{name}` which is not allowed. It either means you"
+            " are trying to save tensors which are reference of each other in which case it's recommended to save"
+            " only the full tensors, and reslice at load time, or simply call `.contiguous()` on your tensor to"
+            " pack it before saving."
+        )
+    if tensor.device.type != "cpu":
+        # Moving tensor to cpu before saving
+        tensor = tensor.to("cpu")
+
+    import ctypes
+
+    import numpy as np
+
+    # When shape is empty (scalar), np.prod returns a float
+    # we need a int for the following calculations
+    length = int(np.prod(tensor.shape).item())
+    bytes_per_item = _SIZE[tensor.dtype]
+
+    total_bytes = length * bytes_per_item
+
+    ptr = tensor.data_ptr()
+    if ptr == 0:
+        return b""
+    newptr = ctypes.cast(ptr, ctypes.POINTER(ctypes.c_ubyte))
+    data = np.ctypeslib.as_array(newptr, (total_bytes,))  # no internal copy
+    if sys.byteorder == "big":
+        NPDTYPES = {
+            torch.int64: np.int64,
+            torch.float32: np.float32,
+            torch.int32: np.int32,
+            # XXX: This is ok because both have the same width
+            torch.bfloat16: np.float16,
+            torch.float16: np.float16,
+            torch.int16: np.int16,
+            torch.uint8: np.uint8,
+            torch.int8: np.int8,
+            torch.bool: bool,
+            torch.float64: np.float64,
+            # XXX: This is ok because both have the same width and byteswap is a no-op anyway
+            _float8_e4m3fn: np.uint8,
+            _float8_e5m2: np.uint8,
+            torch.complex64: np.complex64,
+        }
+        npdtype = NPDTYPES[tensor.dtype]
+        # Not in place as that would potentially modify a live running model
+        data = data.view(npdtype).byteswap(inplace=False)
+    return data.tobytes()
+
+
+def _flatten(tensors: Dict[str, torch.Tensor]) -> Dict[str, Dict[str, Any]]:
+    if not isinstance(tensors, dict):
+        raise ValueError(
+            f"Expected a dict of [str, torch.Tensor] but received {type(tensors)}"
+        )
+
+    invalid_tensors = []
+    for k, v in tensors.items():
+        if not isinstance(v, torch.Tensor):
+            raise ValueError(
+                f"Key `{k}` is invalid, expected torch.Tensor but received {type(v)}"
+            )
+
+        if v.layout != torch.strided:
+            invalid_tensors.append(k)
+    if invalid_tensors:
+        raise ValueError(
+            f"You are trying to save a sparse tensors: `{invalid_tensors}` which this library does not support."
+            " You can make it a dense tensor before saving with `.to_dense()` but be aware this might"
+            " make a much larger file than needed."
+        )
+
+    shared_pointers = _find_shared_tensors(tensors)
+    failing = []
+    for names in shared_pointers:
+        if len(names) > 1:
+            failing.append(names)
+
+    if failing:
+        raise RuntimeError(
+            f"""
+            Some tensors share memory, this will lead to duplicate memory on disk and potential differences when loading them again: {failing}.
+            A potential way to correctly save your model is to use `save_model`.
+            More information at https://huggingface.co/docs/safetensors/torch_shared_tensors
+            """
+        )
+
+    return {
+        k: {
+            "dtype": str(v.dtype).split(".")[-1],
+            "shape": v.shape,
+            "data": _tobytes(v, k),
+        }
+        for k, v in tensors.items()
+    }

source/sentencepiece-0.2.1.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/sentencepiece-0.2.1.dist-info/METADATA

@@ -0,0 +1,251 @@
+Metadata-Version: 2.4
+Name: sentencepiece
+Version: 0.2.1
+Summary: Unsupervised text tokenizer and detokenizer.
+Author-email: Taku Kudo <taku@google.com>
+Project-URL: Homepage, https://github.com/google/sentencepiece
+Classifier: Programming Language :: Python :: 3
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Provides-Extra: testpaths
+Requires-Dist: test; extra == "testpaths"
+
+# SentencePiece Python Wrapper
+
+Python wrapper for SentencePiece. This API will offer the encoding, decoding and training of Sentencepiece.
+
+## Build and Install SentencePiece
+
+For Linux (x64/i686), macOS, and Windows(win32/x64/arm64) environment, you can simply use pip command to install SentencePiece python module.
+
+```
+% pip install sentencepiece
+```
+
+Before building SentencePiece from source on Linux, ensure that the following dependencies are installed.
+
+```
+% sudo apt update
+% sudo apt install -y cmake pkg-config libsentencepiece-dev
+```
+
+To build and install the Python wrapper from source, try the following commands to build and install wheel package.
+
+```
+% git clone https://github.com/google/sentencepiece.git
+% cd sentencepiece
+% mkdir build
+% cd build
+% cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=./root -DSPM_DISABLE_EMBEDDED_DATA=ON
+% make install
+% cd ../python
+% python setup.py bdist_wheel
+% pip install dist/sentencepiece*.whl
+```
+
+If you don’t have write permission to the global site-packages directory or don’t want to install into it, please try:
+
+```
+% python setup.py install --user
+```
+
+For Windows users who want to build from source, you can build and install the Python wrapper using Visual Studio. First, you need to install the `pwsh.exe` (Powershell 7). Use `winget install --id Microsoft.Powershell --source winget` to install directly. Then open the `Developer PowerShell for VS 2022`, and execute the following commands.
+
+```
+git clone https://github.com/google/sentencepiece.git
+cd sentencepiece
+mkdir build
+cd build
+cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=".\root" -DSPM_DISABLE_EMBEDDED_DATA=ON
+cmake --build . --config Release --target install
+cd ../python
+pip install wheel
+python setup.py bdist_wheel
+Get-ChildItem .\dist\sentencepiece*.whl | ForEach-Object { pip install $_.FullName }
+```
+
+## Usage
+
+See [this google colab page](https://github.com/google/sentencepiece/blob/master/python/sentencepiece_python_module_example.ipynb) to run sentencepiece interactively.
+
+### Segmentation
+
+```
+% python
+>>> import sentencepiece as spm
+>>> sp = spm.SentencePieceProcessor(model_file='test/test_model.model')
+
+>>> sp.encode('This is a test')
+[284, 47, 11, 4, 15, 400]
+
+>>> sp.encode(['This is a test', 'Hello world'], out_type=int)
+[[284, 47, 11, 4, 15, 400], [151, 88, 21, 887]]
+
+>>> sp.encode_as_ids(['This is a test', 'Hello world'])
+[[284, 47, 11, 4, 15, 400], [151, 88, 21, 887]]
+
+>>> sp.encode('This is a test', out_type=str)
+['▁This', '▁is', '▁a', '▁', 't', 'est']
+
+>>> sp.encode(['This is a test', 'Hello world'], out_type=str)
+[['▁This', '▁is', '▁a', '▁', 't', 'est'], ['▁He', 'll', 'o', '▁world']]
+
+>>> sp.encode_as_pieces(['This is a test', 'Hello world'])
+[['▁This', '▁is', '▁a', '▁', 't', 'est'], ['▁He', 'll', 'o', '▁world']]
+
+>>> proto = sp.encode('This is a test', out_type='immutable_proto')
+>>> for n in proto.pieces:
+...     print('piece="{}" surface="{}" id={} begin={} end={}'.format(n.piece, n.surface, n.id, n.begin, n.end))
+...
+piece="▁This" surface="This" id=284 begin=0 end=4
+piece="▁is" surface=" is" id=47 begin=4 end=7
+piece="▁a" surface=" a" id=11 begin=7 end=9
+piece="▁" surface=" " id=4 begin=9 end=10
+piece="t" surface="t" id=15 begin=10 end=11
+piece="est" surface="est" id=400 begin=11 end=14
+
+>>> [[x.id for x in proto.pieces], [x.piece for x in proto.pieces], [x.begin for x in proto.pieces], [x.end for x in proto.pieces]]
+[[284, 47, 11, 4, 15, 400], ['▁This', '▁is', '▁a', '▁', 't', 'est'], [0, 4, 7, 9, 10, 11], [4, 7, 9, 10, 11, 14]]
+
+>>> proto2 = sp.encode_as_immutable_proto('This is a test')
+>>> proto2 == proto
+True
+
+>>> for _ in range(10):
+...     sp.encode('This is a test', out_type=str, enable_sampling=True, alpha=0.1, nbest_size=-1)
+...
+['▁', 'This', '▁', 'is', '▁a', '▁', 't', 'e', 'st']
+['▁T', 'h', 'i', 's', '▁is', '▁a', '▁', 'te', 's', 't']
+['▁T', 'h', 'is', '▁', 'is', '▁', 'a', '▁', 't', 'est']
+['▁', 'This', '▁is', '▁', 'a', '▁', 't', 'e', 'st']
+['▁', 'This', '▁', 'is', '▁', 'a', '▁', 't', 'e', 's', 't']
+['▁This', '▁is', '▁a', '▁', 'te', 's', 't']
+['▁This', '▁is', '▁', 'a', '▁', 't', 'e', 'st']
+['▁', 'T', 'h', 'is', '▁', 'is', '▁', 'a', '▁', 'te', 'st']
+['▁', 'This', '▁', 'i', 's', '▁a', '▁', 't', 'e', 'st']
+['▁This', '▁', 'is', '▁a', '▁', 't', 'est']
+
+>> sp.nbest_encode('This is a test', nbest_size=5, out_type=str)
+[['▁This', '▁is', '▁a', '▁', 't', 'est'],
+['▁This', '▁is', '▁a', '▁', 'te', 'st'],
+['▁This', '▁is', '▁a', '▁', 'te', 's', 't'],
+['▁This', '▁is', '▁a', '▁', 't', 'e', 'st'],
+['▁This', '▁is', '▁a', '▁', 't', 'es', 't']]
+
+>>> sp.sample_encode_and_score('This is a test', num_samples=5, alpha=0.1, out_type=str, wor=True)
+[(['▁This', '▁', 'i', 's', '▁a', '▁', 'te', 's', 't'], -3.043105125427246),
+(['▁This', '▁', 'i', 's', '▁a', '▁', 'te', 'st'], -2.8475849628448486),
+(['▁', 'This', '▁is', '▁', 'a', '▁', 'te', 'st'], -3.043248176574707),
+(['▁', 'This', '▁is', '▁a', '▁', 't', 'e', 'st'], -2.87727689743042),
+(['▁', 'This', '▁', 'i', 's', '▁', 'a', '▁', 't', 'est'], -3.6284031867980957)]
+
+>>> sp.decode([284, 47, 11, 4, 15, 400])
+'This is a test'
+
+>>> sp.decode([[284, 47, 11, 4, 15, 400], [151, 88, 21, 887]])
+['This is a test', 'Hello world']
+
+>>> proto = sp.decode([284, 47, 11, 4, 15, 400], out_type='immutable_proto')
+>>> proto.text
+'This is a test'
+
+>>> sp.decode(['▁', 'This', '▁', 'is', '▁a', '▁', 't', 'e', 'st'])
+'This is a test'
+
+>>> sp.decode([['▁This', '▁is', '▁a', '▁', 't', 'est'], ['▁He', 'll', 'o', '▁world']])
+['This is a test', 'Hello world']
+
+>>> sp.get_piece_size()
+1000
+
+>>> sp.id_to_piece(2)
+'</s>'
+
+>>> sp.id_to_piece([2, 3, 4])
+['</s>', '\r', '▁']
+
+>>> sp.piece_to_id('<s>')
+1
+
+>>> sp.piece_to_id(['</s>', '\r', '▁'])
+[2, 3, 4]
+
+>>> len(sp)
+1000
+
+>>> sp['</s>']
+2
+```
+
+### Model Training
+
+Training is performed by passing parameters of [spm_train](https://github.com/google/sentencepiece#train-sentencepiece-model) to SentencePieceTrainer.train() function.
+
+```
+>>> import sentencepiece as spm
+>>> spm.SentencePieceTrainer.train(input='test/botchan.txt', model_prefix='m', vocab_size=1000, user_defined_symbols=['foo', 'bar'])
+sentencepiece_trainer.cc(73) LOG(INFO) Starts training with :
+trainer_spec {
+  input: test/botchan.txt
+  .. snip
+unigram_model_trainer.cc(500) LOG(INFO) EM sub_iter=1 size=1188 obj=10.2839 num_tokens=32182 num_tokens/piece=27.0892
+unigram_model_trainer.cc(500) LOG(INFO) EM sub_iter=0 size=1100 obj=10.4269 num_tokens=33001 num_tokens/piece=30.0009
+unigram_model_trainer.cc(500) LOG(INFO) EM sub_iter=1 size=1100 obj=10.4069 num_tokens=33002 num_tokens/piece=30.0018
+trainer_interface.cc(595) LOG(INFO) Saving model: m.model
+trainer_interface.cc(619) LOG(INFO) Saving vocabs: m.vocab
+>>>
+```
+
+### Training without local filesystem
+
+Sentencepiece trainer can receive any iterable object to feed training sentences. You can also pass a file object (instance with write() method) to emit the output model to any devices. These features are useful to run sentencepiece on environment that have limited access to the local file system (e.g., Google colab.)
+
+```
+import urllib.request
+import io
+import sentencepiece as spm
+
+# Loads model from URL as iterator and stores the model to BytesIO.
+model = io.BytesIO()
+with urllib.request.urlopen(
+    'https://raw.githubusercontent.com/google/sentencepiece/master/data/botchan.txt'
+) as response:
+  spm.SentencePieceTrainer.train(
+      sentence_iterator=response, model_writer=model, vocab_size=1000)
+
+# Serialize the model as file.
+# with open('out.model', 'wb') as f:
+#   f.write(model.getvalue())
+
+# Directly load the model from serialized model.
+sp = spm.SentencePieceProcessor(model_proto=model.getvalue())
+print(sp.encode('this is test'))
+```
+
+### Free Threading support
+Experimental support for no-GIL/Free-Threading has been introduced since v0.2.1. For more details, please refer to [this page](https://py-free-threading.github.io.).
+This operates similarly to how [NumPy](https://numpy.org/devdocs/reference/thread_safety.html#free-threaded-python) handles it.
+
+The C++ library's const and static methods, e.g., encode(), decode() and train(), are designed to work in a non-GIL environment.
+However, non-const methods, e.g., load(), may have potential data race issues, so please ensure you implement appropriate locks beforehand.
+
+While this limitation might be removed in the future, please note that it's not a simple fix, as it would require additional shared locks in C++.

source/sentencepiece-0.2.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+sentencepiece-0.2.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+sentencepiece-0.2.1.dist-info/METADATA,sha256=zgounFmc0GCnvD3_sYvlz8PV8pNTPj7k6HDq7wpVAP0,10003
+sentencepiece-0.2.1.dist-info/RECORD,,
+sentencepiece-0.2.1.dist-info/WHEEL,sha256=1crAxrAH5rUbvWUY1UR0ly3o7KnT1jo0_98V8RY5-FM,152
+sentencepiece-0.2.1.dist-info/top_level.txt,sha256=NIXVKmsq-xdZ5KROMHHV0gkLA3fvbNTb1g1KtgiYFOk,130
+sentencepiece/__init__.py,sha256=NQT0kP3jkjEmRRIyK_jsKCug6L9zQi2vIPTWQin8E2w,49422
+sentencepiece/__pycache__/__init__.cpython-312.pyc,,
+sentencepiece/__pycache__/_version.cpython-312.pyc,,
+sentencepiece/__pycache__/sentencepiece_model_pb2.cpython-312.pyc,,
+sentencepiece/__pycache__/sentencepiece_pb2.cpython-312.pyc,,
+sentencepiece/_sentencepiece.cpython-312-x86_64-linux-gnu.so,sha256=z2sCOJaqHJkCZEXvwrXL07ipfGg53Q_fsrynV_vtlw0,2005912
+sentencepiece/_version.py,sha256=PmcQ2PI2oP8irnLtJLJby2YfW6sBvLAmL-VpABzTqwc,22
+sentencepiece/package_data/nfkc.bin,sha256=UvEQKP-KffPgCdlKi2pU1KihcTLvtMzByaCkHkMr2R4,240008
+sentencepiece/package_data/nfkc_cf.bin,sha256=YIM-wRIBRGZZw1ScGDsY8CTEAHYozGs6TpGuAHaXuCY,247028
+sentencepiece/package_data/nmt_nfkc.bin,sha256=eTcsQTicK5spvBcQF6tUAONS3r1oawJnCkK-xwkBUHQ,240007
+sentencepiece/package_data/nmt_nfkc_cf.bin,sha256=IsKSx29QN5XzDIXXnTCn9XL_9PSeADktLWD0-T6UGh4,247027
+sentencepiece/sentencepiece.i,sha256=Hfv8AHFOJEfDfElVYIhoz29W7rV1VJ0Z13aP7S7ck6M,72647
+sentencepiece/sentencepiece_model_pb2.py,sha256=LawEwmdUiIU1T9HcYu-rNEVTFcwAh9i-qavMMsg9riE,6257
+sentencepiece/sentencepiece_pb2.py,sha256=_ZgnXOkpoScMXbJ-8BMKn2Q97BbMOH9Hz-L7JFMcJro,1753
+sentencepiece/sentencepiece_wrap.cxx,sha256=XlbUFs7s48i3i_nhka9U_b41Xpv_eeSD9U_uxO742Y0,381494

source/sentencepiece-0.2.1.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: false
+Tag: cp312-cp312-manylinux_2_27_x86_64
+Tag: cp312-cp312-manylinux_2_28_x86_64
+

source/sentencepiece-0.2.1.dist-info/top_level.txt

@@ -0,0 +1,5 @@
+sentencepiece
+sentencepiece/__init__
+sentencepiece/_version
+sentencepiece/sentencepiece_model_pb2
+sentencepiece/sentencepiece_pb2

source/sentencepiece/__init__.py

@@ -0,0 +1,1230 @@
+# This file was automatically generated by SWIG (https://www.swig.org).
+# Version 4.3.0
+#
+# Do not make changes to this file unless you know what you are doing - modify
+# the SWIG interface file instead.
+
+from sys import version_info as _swig_python_version_info
+# Import the low-level C/C++ module
+if __package__ or "." in __name__:
+    from . import _sentencepiece
+else:
+    import _sentencepiece
+
+try:
+    import builtins as __builtin__
+except ImportError:
+    import __builtin__
+
+def _swig_repr(self):
+    try:
+        strthis = "proxy of " + self.this.__repr__()
+    except __builtin__.Exception:
+        strthis = ""
+    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
+
+
+def _swig_setattr_nondynamic_instance_variable(set):
+    def set_instance_attr(self, name, value):
+        if name == "this":
+            set(self, name, value)
+        elif name == "thisown":
+            self.this.own(value)
+        elif hasattr(self, name) and isinstance(getattr(type(self), name), property):
+            set(self, name, value)
+        else:
+            raise AttributeError("You cannot add instance attributes to %s" % self)
+    return set_instance_attr
+
+
+def _swig_setattr_nondynamic_class_variable(set):
+    def set_class_attr(cls, name, value):
+        if hasattr(cls, name) and not isinstance(getattr(cls, name), property):
+            set(cls, name, value)
+        else:
+            raise AttributeError("You cannot add class attributes to %s" % cls)
+    return set_class_attr
+
+
+def _swig_add_metaclass(metaclass):
+    """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass"""
+    def wrapper(cls):
+        return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy())
+    return wrapper
+
+
+class _SwigNonDynamicMeta(type):
+    """Meta class to enforce nondynamic attributes (no new attributes) for a class"""
+    __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__)
+
+
+class ImmutableSentencePieceText_ImmutableSentencePiece(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText_ImmutableSentencePiece())
+    __swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText_ImmutableSentencePiece
+
+    def _piece(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__piece(self)
+
+    def _surface(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__surface(self)
+
+    def _id(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__id(self)
+
+    def _begin(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__begin(self)
+
+    def _end(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__end(self)
+
+    def _surface_as_bytes(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes(self)
+
+    def _piece_as_bytes(self):
+        return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes(self)
+
+    piece = property(_piece)
+    piece_as_bytes = property(_piece_as_bytes)
+    surface = property(_surface)
+    surface_as_bytes = property(_surface_as_bytes)
+    id = property(_id)
+    begin = property(_begin)
+    end = property(_end)
+
+    def __str__(self):
+      return ('piece: \"{}\"\n'
+              'id: {}\n'
+              'surface: \"{}\"\n'
+              'begin: {}\n'
+              'end: {}\n').format(self.piece, self.id, self.surface,
+                                  self.begin, self.end)
+
+    def __eq__(self, other):
+      return self.piece == other.piece and self.id == other.id and self.surface == other.surface and self.begin == other.begin and self.end == other.end
+
+    def __hash__(self):
+      return hash(str(self))
+
+    __repr__ = __str__
+
+
+# Register ImmutableSentencePieceText_ImmutableSentencePiece in _sentencepiece:
+_sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swigregister(ImmutableSentencePieceText_ImmutableSentencePiece)
+class ImmutableSentencePieceText(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _sentencepiece.ImmutableSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText())
+    __swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText
+
+    def _pieces_size(self):
+        return _sentencepiece.ImmutableSentencePieceText__pieces_size(self)
+
+    def _pieces(self, index):
+        return _sentencepiece.ImmutableSentencePieceText__pieces(self, index)
+
+    def _text(self):
+        return _sentencepiece.ImmutableSentencePieceText__text(self)
+
+    def _score(self):
+        return _sentencepiece.ImmutableSentencePieceText__score(self)
+
+    def SerializeAsString(self):
+        return _sentencepiece.ImmutableSentencePieceText_SerializeAsString(self)
+
+    def _text_as_bytes(self):
+        return _sentencepiece.ImmutableSentencePieceText__text_as_bytes(self)
+
+    text = property(_text)
+    text_as_bytes = property(_text_as_bytes)
+    score = property(_score)
+
+    class ImmutableSentencePieceIterator:
+      def __init__(self, proto):
+        self.proto = proto
+        self.len = self.proto._pieces_size()
+
+      def __len__(self):
+        return self.len
+
+      def __getitem__(self, index):
+        if isinstance(index, slice):
+          return [self.proto._pieces(i) for i in range(self.len)][index.start:index.stop:index.step]
+        if index < 0:
+          index = index + self.len
+        if index < 0 or index >= self.len:
+          raise IndexError('piece index is out of range')
+        return self.proto._pieces(index)
+
+      def __str__(self):
+        return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self])
+
+      __repr__ = __str__
+
+    @property
+    def pieces(self):
+      return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self)
+
+    def __eq__(self, other):
+      return self.SerializeAsString() == other.SerializeAsString()
+
+    def __hash__(self):
+      return hash(self.SerializeAsString())
+
+    def __str__(self):
+      return ('text: \"{}\"\n'
+              'score: {}\n'
+              '{}').format(self.text, self.score,
+                           '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces]))
+
+    __repr__ = __str__
+
+
+# Register ImmutableSentencePieceText in _sentencepiece:
+_sentencepiece.ImmutableSentencePieceText_swigregister(ImmutableSentencePieceText)
+class ImmutableNBestSentencePieceText(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _sentencepiece.ImmutableNBestSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableNBestSentencePieceText())
+    __swig_destroy__ = _sentencepiece.delete_ImmutableNBestSentencePieceText
+
+    def _nbests_size(self):
+        return _sentencepiece.ImmutableNBestSentencePieceText__nbests_size(self)
+
+    def _nbests(self, index):
+        return _sentencepiece.ImmutableNBestSentencePieceText__nbests(self, index)
+
+    def SerializeAsString(self):
+        return _sentencepiece.ImmutableNBestSentencePieceText_SerializeAsString(self)
+
+    class ImmutableSentencePieceTextIterator:
+      def __init__(self, proto):
+        self.proto = proto
+        self.len = self.proto._nbests_size()
+
+      def __len__(self):
+        return self.len
+
+      def __getitem__(self, index):
+        if isinstance(index, slice):
+          return [self.proto._nbests(i) for i in range(self.len)][index.start:index.stop:index.step]
+        if index < 0:
+          index = index + self.len
+        if index < 0 or index >= self.len:
+          raise IndexError('nbests index is out of range')
+        return self.proto._nbests(index)
+
+      def __str__(self):
+        return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self])
+
+      __repr__ = __str__
+
+    @property
+    def nbests(self):
+      return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self)
+
+    def __eq__(self, other):
+      return self.SerializeAsString() == other.SerializeAsString()
+
+    def __hash__(self):
+      return hash(self.SerializeAsString())
+
+    def __str__(self):
+      return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests])
+
+    __repr__ = __str__
+
+
+# Register ImmutableNBestSentencePieceText in _sentencepiece:
+_sentencepiece.ImmutableNBestSentencePieceText_swigregister(ImmutableNBestSentencePieceText)
+class SentencePieceProcessor(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _sentencepiece.SentencePieceProcessor_swiginit(self, _sentencepiece.new_SentencePieceProcessor())
+    __swig_destroy__ = _sentencepiece.delete_SentencePieceProcessor
+
+    def LoadFromSerializedProto(self, serialized):
+        return _sentencepiece.SentencePieceProcessor_LoadFromSerializedProto(self, serialized)
+
+    def SetEncodeExtraOptions(self, extra_option):
+        return _sentencepiece.SentencePieceProcessor_SetEncodeExtraOptions(self, extra_option)
+
+    def SetDecodeExtraOptions(self, extra_option):
+        return _sentencepiece.SentencePieceProcessor_SetDecodeExtraOptions(self, extra_option)
+
+    def SetVocabulary(self, valid_vocab):
+        return _sentencepiece.SentencePieceProcessor_SetVocabulary(self, valid_vocab)
+
+    def ResetVocabulary(self):
+        return _sentencepiece.SentencePieceProcessor_ResetVocabulary(self)
+
+    def LoadVocabulary(self, filename, threshold):
+        return _sentencepiece.SentencePieceProcessor_LoadVocabulary(self, filename, threshold)
+
+    def CalculateEntropy(self, *args):
+        return _sentencepiece.SentencePieceProcessor_CalculateEntropy(self, *args)
+
+    def GetPieceSize(self):
+        return _sentencepiece.SentencePieceProcessor_GetPieceSize(self)
+
+    def PieceToId(self, piece):
+        return _sentencepiece.SentencePieceProcessor_PieceToId(self, piece)
+
+    def IdToPiece(self, id):
+        return _sentencepiece.SentencePieceProcessor_IdToPiece(self, id)
+
+    def GetScore(self, id):
+        return _sentencepiece.SentencePieceProcessor_GetScore(self, id)
+
+    def IsUnknown(self, id):
+        return _sentencepiece.SentencePieceProcessor_IsUnknown(self, id)
+
+    def IsControl(self, id):
+        return _sentencepiece.SentencePieceProcessor_IsControl(self, id)
+
+    def IsUnused(self, id):
+        return _sentencepiece.SentencePieceProcessor_IsUnused(self, id)
+
+    def IsByte(self, id):
+        return _sentencepiece.SentencePieceProcessor_IsByte(self, id)
+
+    def unk_id(self):
+        return _sentencepiece.SentencePieceProcessor_unk_id(self)
+
+    def bos_id(self):
+        return _sentencepiece.SentencePieceProcessor_bos_id(self)
+
+    def eos_id(self):
+        return _sentencepiece.SentencePieceProcessor_eos_id(self)
+
+    def pad_id(self):
+        return _sentencepiece.SentencePieceProcessor_pad_id(self)
+
+    def serialized_model_proto(self):
+        return _sentencepiece.SentencePieceProcessor_serialized_model_proto(self)
+
+    def LoadFromFile(self, arg):
+        return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
+
+    def _EncodeAsIds(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsIds(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsPieces(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsPieces(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsSerializedProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsSerializedProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsImmutableProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsImmutableProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsIdsBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsIdsBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsPiecesBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsPiecesBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsSerializedProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsSerializedProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _EncodeAsImmutableProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__EncodeAsImmutableProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _DecodeIds(self, ids):
+        return _sentencepiece.SentencePieceProcessor__DecodeIds(self, ids)
+
+    def _DecodeIdsAsBytes(self, ids):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsBytes(self, ids)
+
+    def _DecodePieces(self, pieces):
+        return _sentencepiece.SentencePieceProcessor__DecodePieces(self, pieces)
+
+    def _DecodeIdsAsSerializedProto(self, ids):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsSerializedProto(self, ids)
+
+    def _DecodePiecesAsSerializedProto(self, pieces):
+        return _sentencepiece.SentencePieceProcessor__DecodePiecesAsSerializedProto(self, pieces)
+
+    def _DecodeIdsAsImmutableProto(self, ids):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsImmutableProto(self, ids)
+
+    def _DecodePiecesAsImmutableProto(self, pieces):
+        return _sentencepiece.SentencePieceProcessor__DecodePiecesAsImmutableProto(self, pieces)
+
+    def _DecodeIdsBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsBatch(self, ins, num_threads)
+
+    def _DecodeIdsAsBytesBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsBytesBatch(self, ins, num_threads)
+
+    def _DecodeIdsAsSerializedProtoBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(self, ins, num_threads)
+
+    def _DecodeIdsAsImmutableProtoBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch(self, ins, num_threads)
+
+    def _DecodePiecesBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodePiecesBatch(self, ins, num_threads)
+
+    def _DecodePiecesAsSerializedProtoBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(self, ins, num_threads)
+
+    def _DecodePiecesAsImmutableProtoBatch(self, ins, num_threads):
+        return _sentencepiece.SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch(self, ins, num_threads)
+
+    def _NBestEncodeAsIds(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__NBestEncodeAsIds(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _NBestEncodeAsPieces(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__NBestEncodeAsPieces(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _NBestEncodeAsSerializedProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__NBestEncodeAsSerializedProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _NBestEncodeAsImmutableProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__NBestEncodeAsImmutableProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _SampleEncodeAndScoreAsIds(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsIds(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _SampleEncodeAndScoreAsPieces(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsPieces(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _SampleEncodeAndScoreAsSerializedProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _SampleEncodeAndScoreAsImmutableProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece):
+        return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece)
+
+    def _Normalize(self, text):
+        return _sentencepiece.SentencePieceProcessor__Normalize(self, text)
+
+    def _NormalizeWithOffsets(self, text):
+        return _sentencepiece.SentencePieceProcessor__NormalizeWithOffsets(self, text)
+
+    def _CalculateEntropy(self, text, alpha):
+        return _sentencepiece.SentencePieceProcessor__CalculateEntropy(self, text, alpha)
+
+    def _CalculateEntropyBatch(self, ins, alpha, num_threads):
+        return _sentencepiece.SentencePieceProcessor__CalculateEntropyBatch(self, ins, alpha, num_threads)
+
+    def _OverrideNormalizerSpec(self, args):
+        return _sentencepiece.SentencePieceProcessor__OverrideNormalizerSpec(self, args)
+
+    def Init(self,
+             model_file=None,
+             model_proto=None,
+             out_type=int,
+             add_bos=False,
+             add_eos=False,
+             reverse=False,
+             emit_unk_piece=False,
+             enable_sampling=False,
+             nbest_size=-1,
+             alpha=0.1,
+             num_threads=-1):
+      """Initialzie sentencepieceProcessor.
+
+      Args:
+        model_file: The sentencepiece model file path.
+        model_proto: The sentencepiece model serialized proto.
+        out_type: output type. int or str.
+        add_bos: Add <s> to the result (Default = false)
+        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
+          reversing (if enabled).
+        reverse: Reverses the tokenized sequence (Default = false)
+        emit_unk_piece: Emits the unk literal string (Default = false)
+        nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
+                    nbest_size = {0,1}: No sampling is performed.
+                    nbest_size > 1: samples from the nbest_size results.
+                    nbest_size < 0: assuming that nbest_size is infinite and samples
+                      from the all hypothesis (lattice) using
+                      forward-filtering-and-backward-sampling algorithm.
+        alpha: Soothing parameter for unigram sampling, and dropout probability of
+               merge operations for BPE-dropout.
+        num_threads: number of threads in batch processing (Default = -1, auto-detected)
+      """
+
+      _sentencepiece_processor_init_native(self)
+      self._out_type = out_type
+      self._add_bos = add_bos
+      self._add_eos = add_eos
+      self._reverse = reverse
+      self._emit_unk_piece = emit_unk_piece
+      self._enable_sampling = enable_sampling
+      self._nbest_size = nbest_size
+      self._alpha = alpha
+      self._num_threads = num_threads
+      if model_file or model_proto:
+        self.Load(model_file=model_file, model_proto=model_proto)
+
+
+    def Encode(self,
+               input,
+               out_type=None,
+               add_bos=None,
+               add_eos=None,
+               reverse=None,
+               emit_unk_piece=None,
+               enable_sampling=None,
+               nbest_size=None,
+               alpha=None,
+               num_threads=None):
+      """Encode text input to segmented ids or tokens.
+
+        Args:
+        input: input string. accepsts list of string.
+        out_type: output type. int or str.
+        add_bos: Add <s> to the result (Default = false)
+        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
+                 reversing (if enabled).
+        reverse: Reverses the tokenized sequence (Default = false)
+        emit_unk_piece: Emits the unk literal string (Default = false)
+        nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
+                    nbest_size = {0,1}: No sampling is performed.
+                    nbest_size > 1: samples from the nbest_size results.
+                    nbest_size < 0: assuming that nbest_size is infinite and samples
+                    from the all hypothesis (lattice) using
+                    forward-filtering-and-backward-sampling algorithm.
+        alpha: Soothing parameter for unigram sampling, and merge probability for
+               BPE-dropout (probablity 'p' in BPE-dropout paper).
+        num_threads: the number of threads used in the batch processing (Default = -1).
+      """
+
+      if out_type is None:
+        out_type = self._out_type
+      if add_bos is None:
+        add_bos = self._add_bos
+      if add_eos is None:
+        add_eos = self._add_eos
+      if reverse is None:
+        reverse = self._reverse
+      if emit_unk_piece is None:
+        emit_unk_piece = self._emit_unk_piece
+      if enable_sampling is None:
+        enable_sampling = self._enable_sampling
+      if nbest_size is None:
+        nbest_size = self._nbest_size
+      if alpha is None:
+        alpha = self._alpha
+      if num_threads is None:
+        num_threads = self._num_threads
+
+      if enable_sampling == True and (nbest_size is None or nbest_size == 0 or
+                                      nbest_size == 1 or alpha is None):
+        raise RuntimeError(
+            'When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", '
+            'and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. '
+            'when "nbest_size = -1" , this method samples from all candidates on the lattice '
+            'instead of nbest segmentations.'
+        )
+
+      if num_threads is None or type(num_threads) is not int:
+        raise RuntimeError('num_threads must be int')
+
+      if type(input) is list:
+        if out_type is int:
+          return self._EncodeAsIdsBatch(input, num_threads, enable_sampling, nbest_size,
+                                        alpha, add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type is str:
+          return self._EncodeAsPiecesBatch(input, num_threads, enable_sampling, nbest_size,
+                                           alpha, add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type == 'serialized_proto' or out_type == 'proto':
+          return self._EncodeAsSerializedProtoBatch(input, num_threads, enable_sampling, nbest_size,
+                                                    alpha, add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type == 'immutable_proto':
+          return self._EncodeAsImmutableProtoBatch(input, num_threads, enable_sampling, nbest_size,
+                                                   alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+      if out_type is int:
+        return self._EncodeAsIds(input, enable_sampling, nbest_size,
+                                 alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type is str:
+        return self._EncodeAsPieces(input, enable_sampling, nbest_size,
+                                    alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'serialized_proto' or out_type == 'proto':
+        return self._EncodeAsSerializedProto(input, enable_sampling, nbest_size,
+                                             alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'immutable_proto':
+        return self._EncodeAsImmutableProto(input, enable_sampling, nbest_size,
+                                            alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+      raise RuntimeError('unknown out_type={}'.format(out_type))
+      return None
+
+
+    def EncodeAsPieces(self, input, **kwargs):
+      return self.Encode(input=input, out_type=str, **kwargs)
+
+
+    def EncodeAsIds(self, input, **kwargs):
+      return self.Encode(input=input, out_type=int, **kwargs)
+
+
+    def EncodeAsSerializedProto(self, input, **kwargs):
+      return self.Encode(input=input, out_type='serialized_proto', **kwargs)
+
+
+    def EncodeAsImmutableProto(self, input, **kwargs):
+      return self.Encode(input=input, out_type='immutable_proto', **kwargs)
+
+
+    def SampleEncodeAsPieces(self, input, nbest_size=None, alpha=None, **kwargs):
+      return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                         out_type=str, enable_sampling=True, **kwargs)
+
+
+    def SampleEncodeAsIds(self, input, nbest_size=None, alpha=None,**kwargs):
+      return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                         out_type=int, enable_sampling=True, **kwargs)
+
+
+    def SampleEncodeAsSerializedProto(self, input, nbest_size=None, alpha=None, **kwargs):
+      return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                         out_type='serialized_proto', enable_sampling=True, **kwargs)
+
+
+    def SampleEncodeAsImmutableProto(self, input, nbest_size=None, alpha=None, **kwargs):
+      return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                         out_type='immutable_proto', enable_sampling=True, **kwargs)
+
+
+    def NBestEncode(self,
+                    input,
+                    out_type=None,
+                    add_bos=None,
+                    add_eos=None,
+                    reverse=None,
+                    emit_unk_piece=None,
+                    nbest_size=None):
+      """NBestEncode text input to segmented ids or tokens.
+
+        Args:
+        input: input string. accepsts list of string.
+        out_type: output type. int or str.
+        add_bos: Add <s> to the result (Default = false)
+        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
+        reverse: Reverses the tokenized sequence (Default = false)
+        emit_unk_piece: Emits the unk literal string (Default = false)
+        nbest_size: nbest size
+      """
+
+      if out_type is None:
+        out_type = self._out_type
+      if add_bos is None:
+        add_bos = self._add_bos
+      if add_eos is None:
+        add_eos = self._add_eos
+      if reverse is None:
+        reverse = self._reverse
+      if emit_unk_piece is None:
+        emit_unk_piece = self._emit_unk_piece
+      if nbest_size is None:
+        nbest_size = self._nbest_size
+
+      if nbest_size <= 0:
+        nbest_size=1
+
+      def _encode(text):
+        if out_type is int:
+          return self._NBestEncodeAsIds(text, nbest_size,
+                                        add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type is str:
+          return self._NBestEncodeAsPieces(text, nbest_size,
+                                           add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type == 'serialized_proto' or out_type == 'proto':
+          return self._NBestEncodeAsSerializedProto(text, nbest_size,
+                                                    add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type == 'immutable_proto':
+          return self._NBestEncodeAsImmutableProto(text, nbest_size,
+                                                   add_bos, add_eos, reverse, emit_unk_piece)
+
+        raise RuntimeError('unknown out_type')
+
+      if type(input) is list:
+        return [_encode(n) for n in input]
+
+      return _encode(input)
+
+
+    def NBestEncodeAsPieces(self, input, nbest_size=None, **kwargs):
+      return self.NBestEncode(input=input, nbest_size=nbest_size,
+                              out_type=str, **kwargs)
+
+
+    def NBestEncodeAsIds(self, input, nbest_size=None, **kwargs):
+      return self.NBestEncode(input=input, nbest_size=nbest_size,
+                              out_type=int, **kwargs)
+
+
+    def NBestEncodeAsSerializedProto(self, input, nbest_size=None, **kwargs):
+      return self.NBestEncode(input=input, nbest_size=nbest_size,
+                              out_type='serialized_proto', **kwargs)
+
+
+    def NBestEncodeAsImmutableProto(self, input, nbest_size=None, **kwargs):
+      return self.NBestEncode(input=input, nbest_size=nbest_size,
+                              out_type='immutable_proto', **kwargs)
+
+
+    def SampleEncodeAndScore(self,
+                             input,
+                             out_type=None,
+                             add_bos=None,
+                             add_eos=None,
+                             reverse=None,
+                             emit_unk_piece=None,
+                             num_samples=None,
+                             alpha=None,
+                             wor=None,
+                             include_best=None):
+      """SampleEncodeAndScore text input to segmented ids or tokens.
+
+        Args:
+        input: input string. accepsts list of string.
+        out_type: output type. int or str or 'serialized_proto' or 'immutable_proto'
+        add_bos: Add <s> to the result (Default = false)
+        add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
+        reverse: Reverses the tokenized sequence (Default = false)
+        emit_unk_piece: Emits the unk literal string (Default = false)
+        num_samples: How many samples to return (Default = 1)
+        alpha: inverse temperature for sampling
+        wor: whether to sample without replacement (Default = false)
+        include_best: whether to include the best tokenization, requires wor=True (Default = false)
+      """
+
+      if out_type is None:
+        out_type = self._out_type
+      if add_bos is None:
+        add_bos = self._add_bos
+      if add_eos is None:
+        add_eos = self._add_eos
+      if reverse is None:
+        reverse = self._reverse
+      if emit_unk_piece is None:
+        emit_unk_piece = self._emit_unk_piece
+      if num_samples is None:
+        num_samples = 1
+      if alpha is None:
+        alpha = 1.
+      if wor is None:
+        wor = False
+      if include_best is None:
+        include_best = False
+
+      if num_samples <= 0:
+        raise RuntimeError('num_examples must be positive')
+
+      if include_best and not wor:
+        raise RuntimeError('When include_best is True, We must specify "wor = True".')
+
+
+      def _encode(text):
+        if out_type is int:
+          return self._SampleEncodeAndScoreAsIds(text, num_samples, alpha, wor, include_best,
+                                                 add_bos, add_eos, reverse, emit_unk_piece)
+        if out_type is str:
+          return self._SampleEncodeAndScoreAsPieces(text, num_samples, alpha, wor, include_best,
+                                                    add_bos, add_eos, reverse, emit_unk_piece)
+
+        if out_type == 'serialized_proto' or out_type == 'proto':
+          return self._SampleEncodeAndScoreAsSerializedProto(text, num_samples, alpha, wor, include_best,
+                                                             add_bos, add_eos, reverse, emit_unk_piece)
+
+        if out_type == 'immutable_proto':
+          return self._SampleEncodeAndScoreAsImmutableProto(text, num_samples, alpha, wor, include_best,
+                                                            add_bos, add_eos, reverse, emit_unk_piece)
+
+        raise RuntimeError('unknown output type')
+
+
+      if type(input) is list:
+        return [_encode(n) for n in input]
+
+      return _encode(input)
+
+
+    def SampleEncodeAndScoreAsPieces(self, input, num_samples=None, alpha=None, **kwargs):
+      return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                       out_type=str, **kwargs)
+
+
+    def SampleEncodeAndScoreAsIds(self, input, num_samples=None, alpha=None, **kwargs):
+      return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                       out_type=int, **kwargs)
+
+
+    def SampleEncodeAndScoreAsSerializedProto(self, input, num_samples=None, alpha=None, **kwargs):
+      return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                       out_type='serialized_proto', **kwargs)
+
+
+    def SampleEncodeAndScoreAsImmutableProto(self, input, num_samples=None, alpha=None, **kwargs):
+      return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                       out_type='immutable_proto', **kwargs)
+
+
+    def Decode(self, input, out_type=str, num_threads=None):
+      """Decode processed id or token sequences.
+
+      Args:
+        out_type: output type. str, bytes or 'serialized_proto' or 'immutable_proto' (Default = str)
+        num_threads: the number of threads used in the batch processing (Default = -1).
+      """
+
+      if num_threads is None:
+        num_threads = self._num_threads
+
+      if num_threads is None or type(num_threads) is not int:
+        raise RuntimeError('num_threads must be int')
+
+      if not input:
+        return ''
+
+      if out_type is str:
+        if type(input) is int:
+          return self._DecodeIds([input])
+        if type(input) is str:
+          return self._DecodePieces([input])
+
+        if type(input) is list:
+          if len(input) == 0 or type(input[0]) is int:
+            return self._DecodeIds(input)
+          if type(input[0]) is str:
+            return self._DecodePieces(input)
+
+          if type(input[0]) is list:
+            if len(input[0]) == 0 or type(input[0][0]) is int:
+             return self._DecodeIdsBatch(input, num_threads)
+            if type(input[0][0]) is str:
+             return self._DecodePiecesBatch(input, num_threads)
+
+      if out_type is bytes:
+        if type(input) is int:
+          return self._DecodeIdsAsBytes([input])
+        if type(input) is str:
+          return self._DecodePieces([input])
+
+        if type(input) is list:
+          if len(input) == 0 or type(input[0]) is int:
+            return self._DecodeIdsAsBytes(input)
+          if type(input[0]) is str:
+            return self._DecodePieces(input)
+
+          if type(input[0]) is list:
+            if len(input[0]) == 0 or type(input[0][0]) is int:
+             return self._DecodeIdsAsBytesBatch(input, num_threads)
+            if type(input[0][0]) is str:
+             return self._DecodePiecesBatch(input, num_threads)
+
+      if out_type == 'serialized_proto':
+        if type(input) is int:
+          return self._DecodeIdsAsSerializedProto([input])
+        if type(input) is str:
+          return self._DecodePiecesAsSerializedProto([input])
+
+        if type(input) is list:
+          if len(input) == 0 or type(input[0]) is int:
+            return self._DecodeIdsAsSerializedProto(input)
+          if type(input[0]) is str:
+            return self._DecodePiecesAsSerializedProto(input)
+
+          if type(input[0]) is list:
+            if len(input[0]) == 0 or type(input[0][0]) is int:
+             return self._DecodeIdsAsSerializedProtoBatch(input, num_threads)
+            if type(input[0][0]) is str:
+             return self._DecodePiecesAsSerializedProtoBatch(input, num_threads)
+
+
+      if out_type == 'immutable_proto':
+        if type(input) is int:
+          return self._DecodeIdsAsImmutableProto([input])
+        if type(input) is str:
+          return self._DecodePiecesAsImmutableProto([input])
+
+        if type(input) is list:
+          if len(input) == 0 or type(input[0]) is int:
+            return self._DecodeIdsAsImmutableProto(input)
+          if type(input[0]) is str:
+            return self._DecodePiecesAsImmutableProto(input)
+
+          if type(input[0]) is list:
+            if len(input[0]) == 0 or type(input[0][0]) is int:
+             return self._DecodeIdsAsImmutableProtoBatch(input, num_threads)
+            if type(input[0][0]) is str:
+             return self._DecodePiecesAsImmutableProtoBatch(input, num_threads)
+
+
+      raise RuntimeError('unknown output or input type')
+      return None
+
+
+    def DecodePieces(self, input, out_type=str, **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def DecodeIds(self, input, out_type=str, **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def DecodePiecesAsSerializedProto(self, input, out_type='serialized_proto', **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def DecodeIdsAsSerializedProto(self, input, out_type='serialized_proto', **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def DecodePiecesAsImmutableProto(self, input, out_type='immutable_proto', **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def DecodeIdsAsImmutableProto(self, input, out_type='immutable_proto', **kwargs):
+      return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+    def CalculateEntropy(self, input, alpha, num_threads=None):
+      """Calculate sentence entropy"""
+      if type(input) is list:
+        if num_threads is None:
+          num_threads = self._num_threads
+        if num_threads is None or type(num_threads) is not int:
+          raise RuntimeError('num_threads must be int')
+        return self._CalculateEntropyBatch(input, alpha, num_threads)
+
+      return self._CalculateEntropy(input, alpha)
+
+
+    def Normalize(self, input, with_offsets=None):
+      def _normalize(text):
+        if with_offsets:
+          return self._NormalizeWithOffsets(text)
+        return self._Normalize(text)
+
+      if type(input) is list:
+        return [_normalize(x) for x in input]
+      return _normalize(input)
+
+    def OverrideNormalizerSpec(self, **kwargs):
+      new_kwargs = {}
+      for key, value in kwargs.items():
+        new_kwargs[key] = str(value)
+      return self._OverrideNormalizerSpec(new_kwargs)
+
+
+    def piece_size(self):
+      return self.GetPieceSize()
+
+
+    def vocab_size(self):
+      return self.GetPieceSize()
+
+
+    def __getstate__(self):
+      return self.serialized_model_proto()
+
+
+    def __setstate__(self, serialized_model_proto):
+      self.__init__()
+      self.LoadFromSerializedProto(serialized_model_proto)
+
+
+    def __len__(self):
+      return self.GetPieceSize()
+
+
+    def __getitem__(self, piece):
+      return self.PieceToId(piece)
+
+
+    def Load(self, model_file=None, model_proto=None):
+      """Overwride SentencePieceProcessor.Load to support both model_file and model_proto.
+
+      Args:
+        model_file: The sentencepiece model file path.
+        model_proto: The sentencepiece model serialized proto. Either `model_file`
+          or `model_proto` must be set.
+      """
+      if model_file and model_proto:
+        raise RuntimeError('model_file and model_proto must be exclusive.')
+      if model_proto:
+        return self.LoadFromSerializedProto(model_proto)
+      return self.LoadFromFile(model_file)
+
+
+# Register SentencePieceProcessor in _sentencepiece:
+_sentencepiece.SentencePieceProcessor_swigregister(SentencePieceProcessor)
+
+def SetRandomGeneratorSeed(seed):
+    return _sentencepiece.SetRandomGeneratorSeed(seed)
+
+def SetMinLogLevel(v):
+    return _sentencepiece.SetMinLogLevel(v)
+class SentencePieceTrainer(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+
+    def __init__(self, *args, **kwargs):
+        raise AttributeError("No constructor defined")
+    __repr__ = _swig_repr
+
+    @staticmethod
+    def _TrainFromString(arg):
+        return _sentencepiece.SentencePieceTrainer__TrainFromString(arg)
+
+    @staticmethod
+    def _TrainFromMap(args):
+        return _sentencepiece.SentencePieceTrainer__TrainFromMap(args)
+
+    @staticmethod
+    def _TrainFromMap2(args, iter):
+        return _sentencepiece.SentencePieceTrainer__TrainFromMap2(args, iter)
+
+    @staticmethod
+    def _TrainFromMap3(args):
+        return _sentencepiece.SentencePieceTrainer__TrainFromMap3(args)
+
+    @staticmethod
+    def _TrainFromMap4(args, iter):
+        return _sentencepiece.SentencePieceTrainer__TrainFromMap4(args, iter)
+
+    @staticmethod
+    def _Train(arg=None, **kwargs):
+      """Train Sentencepiece model. Accept both kwargs and legacy string arg."""
+      if arg is not None and type(arg) is str:
+        return SentencePieceTrainer._TrainFromString(arg)
+
+      def _encode(value):
+        """Encode value to CSV.."""
+        if type(value) is list:
+          if sys.version_info[0] == 3:
+            f = StringIO()
+          else:
+            f = BytesIO()
+          writer = csv.writer(f, lineterminator='')
+          writer.writerow([str(v) for v in value])
+          return f.getvalue()
+        else:
+          return str(value)
+
+      sentence_iterator = None
+      model_writer = None
+      new_kwargs = {}
+      for key, value in kwargs.items():
+        if key in ['sentence_iterator', 'sentence_reader']:
+          sentence_iterator = value
+        elif key in ['model_writer']:
+          model_writer = value
+        else:
+          new_kwargs[key] = _encode(value)
+
+      if model_writer:
+        if sentence_iterator:
+          model_proto = SentencePieceTrainer._TrainFromMap4(new_kwargs,
+                                                           sentence_iterator)
+        else:
+          model_proto = SentencePieceTrainer._TrainFromMap3(new_kwargs)
+        model_writer.write(model_proto)
+      else:
+        if sentence_iterator:
+          return SentencePieceTrainer._TrainFromMap2(new_kwargs, sentence_iterator)
+        else:
+          return SentencePieceTrainer._TrainFromMap(new_kwargs)
+
+      return None
+
+    @staticmethod
+    def Train(arg=None, logstream=None, **kwargs):
+      with _LogStream(ostream=logstream):
+        SentencePieceTrainer._Train(arg=arg, **kwargs)
+
+
+# Register SentencePieceTrainer in _sentencepiece:
+_sentencepiece.SentencePieceTrainer_swigregister(SentencePieceTrainer)
+class SentencePieceNormalizer(object):
+    thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag")
+    __repr__ = _swig_repr
+
+    def __init__(self):
+        _sentencepiece.SentencePieceNormalizer_swiginit(self, _sentencepiece.new_SentencePieceNormalizer())
+    __swig_destroy__ = _sentencepiece.delete_SentencePieceNormalizer
+
+    def LoadFromSerializedProto(self, serialized):
+        return _sentencepiece.SentencePieceNormalizer_LoadFromSerializedProto(self, serialized)
+
+    def LoadFromRuleTSV(self, filename):
+        return _sentencepiece.SentencePieceNormalizer_LoadFromRuleTSV(self, filename)
+
+    def LoadFromRuleName(self, name):
+        return _sentencepiece.SentencePieceNormalizer_LoadFromRuleName(self, name)
+
+    def serialized_model_proto(self):
+        return _sentencepiece.SentencePieceNormalizer_serialized_model_proto(self)
+
+    def LoadFromFile(self, arg):
+        return _sentencepiece.SentencePieceNormalizer_LoadFromFile(self, arg)
+
+    def _Normalize(self, text):
+        return _sentencepiece.SentencePieceNormalizer__Normalize(self, text)
+
+    def _NormalizeWithOffsets(self, text):
+        return _sentencepiece.SentencePieceNormalizer__NormalizeWithOffsets(self, text)
+
+    def _SetProtoField(self, name, value):
+        return _sentencepiece.SentencePieceNormalizer__SetProtoField(self, name, value)
+
+    def Init(self,
+             model_file=None,
+             model_proto=None,
+             rule_tsv=None,
+             rule_name=None,
+             add_dummy_prefix=False,
+             escape_whitespaces=False,
+             remove_extra_whitespaces=False):
+      """Initialzie sentencePieceNormalizer.
+
+      Args:
+        model_file: The sentencepiece model file path.
+        model_proto: The sentencepiece model serialized proto.
+        rule_tsv: The normalization rule file in TSV format.
+        rule_name: Pre-defined normalization name.
+        add_dummy_prefix: add dummy prefix.
+        escape_whitespaces: escape whitespaces.
+        remove_extra_whitespaces: remove extra whitespaces.
+      """
+
+      _sentencepiece_normalizer_init_native(self)
+
+      if model_file:
+        status = self.LoadFromFile(model_file)
+      elif model_proto:
+        status = self.LoadFromSerializedProto(model_proto)
+      elif rule_tsv:
+        status = self.LoadFromRuleTSV(rule_tsv)
+      elif rule_name:
+        status = self.LoadFromRuleName(rule_name)
+      else:
+        raise RuntimeError('no model is specified')
+
+      if status:
+        self._SetProtoField('add_dummy_prefix', add_dummy_prefix)
+        self._SetProtoField('escape_whitespaces', escape_whitespaces)
+        self._SetProtoField('remove_extra_whitespaces', remove_extra_whitespaces)
+
+    def Normalize(self, input, with_offsets=None):
+      def _normalize(text):
+        if with_offsets:
+          return self._NormalizeWithOffsets(text)
+        return self._Normalize(text)
+
+      if type(input) is list:
+        return [_normalize(x) for x in input]
+      return _normalize(input)
+
+
+    def __getstate__(self):
+      return self.serialized_model_proto()
+
+
+    def __setstate__(self, serialized_model_proto):
+      self.__init__()
+      self.LoadFromSerializedProto(serialized_model_proto)
+
+
+# Register SentencePieceNormalizer in _sentencepiece:
+_sentencepiece.SentencePieceNormalizer_swigregister(SentencePieceNormalizer)
+
+def SetDataDir(data_dir):
+    return _sentencepiece.SetDataDir(data_dir)
+
+
+import re
+import csv
+import sys
+import os
+import importlib.resources
+from io import StringIO
+from io import BytesIO
+
+
+def _add_snake_case(classname):
+  """Added snake_cased method from CammelCased method."""
+
+  snake_map = {}
+  for k, v in classname.__dict__.items():
+    if re.match(r'^[A-Z]+', k):
+      snake = re.sub(r'(?<!^)(?=[A-Z])', '_',
+                     k).lower().replace('n_best', 'nbest')
+      snake_map[snake] = v
+  for k, v in snake_map.items():
+    setattr(classname, k, v)
+
+
+def _batchnize(classname, name):
+  """Enables batch request for the method classname.name."""
+  func = getattr(classname, name, None)
+  def _func(v, n):
+    if type(n) is int and (n < 0 or n >= v.piece_size()):
+      raise IndexError('piece id is out of range.')
+    return func(v, n)
+
+  def _batched_func(self, arg):
+    if type(arg) is list:
+      return [_func(self, n) for n in arg]
+    else:
+      return _func(self, arg)
+
+  setattr(classname, name, _batched_func)
+
+
+_sentencepiece_processor_init_native = SentencePieceProcessor.__init__
+_sentencepiece_normalizer_init_native = SentencePieceNormalizer.__init__
+setattr(SentencePieceProcessor, '__init__', SentencePieceProcessor.Init)
+setattr(SentencePieceNormalizer, '__init__', SentencePieceNormalizer.Init)
+
+SentencePieceProcessor.Tokenize = SentencePieceProcessor.Encode
+SentencePieceProcessor.Detokenize = SentencePieceProcessor.Decode
+
+for m in [
+    'PieceToId', 'IdToPiece', 'GetScore', 'IsUnknown', 'IsControl', 'IsUnused',
+    'IsByte'
+]:
+  _batchnize(SentencePieceProcessor, m)
+
+_add_snake_case(SentencePieceProcessor)
+_add_snake_case(SentencePieceTrainer)
+_add_snake_case(SentencePieceNormalizer)
+set_random_generator_seed = SetRandomGeneratorSeed
+set_min_log_level = SetMinLogLevel
+
+from ._version import __version__
+
+SetDataDir(os.path.join(str(importlib.resources.files('sentencepiece')), 'package_data'))
+
+class _LogStream(object):
+  def __init__(self, ostream=None):
+    self.ostream = ostream
+    if self.ostream is not None:
+      self.orig_stream_fileno = sys.stderr.fileno()
+
+  def __enter__(self):
+    if self.ostream is not None:
+      self.orig_stream_dup = os.dup(self.orig_stream_fileno)
+      os.dup2(self.ostream.fileno(), self.orig_stream_fileno)
+
+  def __exit__(self, type, value, traceback):
+    if self.ostream is not None:
+      os.close(self.orig_stream_fileno)
+      os.dup2(self.orig_stream_dup, self.orig_stream_fileno)
+      os.close(self.orig_stream_dup)
+      self.ostream.close()
+
+

source/sentencepiece/_sentencepiece.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf6b023896aa1c99026445efc2b5cbd3b8a97c6839dd0fdfb2bca757fbed970d
+size 2005912

source/sentencepiece/_version.py

@@ -0,0 +1 @@
+__version__ = '0.2.1'

source/sentencepiece/package_data/nfkc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52f11028ff8a7df3e009d94a8b6a54d4a8a17132efb4ccc1c9a0a41e432bd91e
+size 240008

source/sentencepiece/package_data/nfkc_cf.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60833ec11201446659c3549c183b18f024c4007628cc6b3a4e91ae007697b826
+size 247028

source/sentencepiece/package_data/nmt_nfkc.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79372c41389c2b9b29bc171017ab5400e352debd686b02670a42bec709015074
+size 240007

source/sentencepiece/package_data/nmt_nfkc_cf.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22c292c76f503795f30c85d79d30a7f572fff4f49e00392d2d60f4f93e941a1e
+size 247027

source/sentencepiece/sentencepiece.i

@@ -0,0 +1,2013 @@
+%module sentencepiece
+%include exception.i
+
+%{
+
+#include <atomic>
+#include <iostream>
+#include <algorithm>
+#include <functional>
+#include <limits>
+#include <cmath>
+#include <thread>
+#include <vector>
+#include <sentencepiece_processor.h>
+#include <sentencepiece_trainer.h>
+
+namespace {
+PyObject* kUnicodeInput = reinterpret_cast<PyObject* >(0x1);
+PyObject* kByteInput = reinterpret_cast<PyObject* >(0x2);
+
+using BytesArray = std::vector<sentencepiece::util::bytes>;
+
+inline void ReleaseResultObject(PyObject *obj) {
+  if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) {
+    Py_XDECREF(obj);
+  }
+}
+
+class PyInputString {
+ public:
+  explicit PyInputString(PyObject* obj) {
+    if (PyUnicode_Check(obj)) {
+      str_ = const_cast<char *>(PyUnicode_AsUTF8AndSize(obj, &size_));
+      input_type_ = kUnicodeInput;
+    } else if (PyBytes_Check(obj)) {
+      PyBytes_AsStringAndSize(obj, &str_, &size_);
+      input_type_ = kByteInput;
+    } else {
+      str_ = nullptr;
+    }
+  }
+  absl::string_view str() const { return absl::string_view(data(), size()); }
+  const char* data() const { return str_; }
+  Py_ssize_t size() const { return size_; }
+  bool IsAvalable() const { return str_ != nullptr; }
+  PyObject *input_type() const { return input_type_; }
+
+  static bool IsUnicode(PyObject *resultobj) {
+    return (resultobj == nullptr || resultobj == kUnicodeInput);
+  }
+
+ private:
+  PyObject* input_type_ = nullptr;
+  char* str_ = nullptr;
+  Py_ssize_t size_ = 0;
+};
+
+PyObject* MakePyOutputString(const std::string& output,
+                             PyObject *resultobj) {
+  if (PyInputString::IsUnicode(resultobj)) {
+    return PyUnicode_FromStringAndSize(output.data(), output.size());
+  }
+  return PyBytes_FromStringAndSize(output.data(), output.size());
+}
+
+PyObject* MakePyOutputBytes(const sentencepiece::util::bytes& output) {
+  return PyBytes_FromStringAndSize(output.data(), output.size());
+}
+
+int ToSwigError(sentencepiece::util::StatusCode code) {
+  switch (code) {
+    case sentencepiece::util::StatusCode::kNotFound:
+      return SWIG_IOError;
+    case sentencepiece::util::StatusCode::kOutOfRange:
+      return SWIG_IndexError;
+    case sentencepiece::util::StatusCode::kInvalidArgument:
+      return SWIG_SyntaxError;
+    default:
+      return SWIG_RuntimeError;
+  }
+  return SWIG_RuntimeError;
+}
+
+class PySentenceIterator : public sentencepiece::SentenceIterator {
+  public:
+  PySentenceIterator(PyObject *iter) : iter_(iter) {
+    item_ = PyIter_Next(iter_);
+    CopyValue();
+  }
+
+  ~PySentenceIterator() {
+   // Py_XDECREF(iter_);
+  }
+
+  bool done() const override {
+    return item_ == nullptr;
+  }
+
+  void Next() override {
+    item_ = PyIter_Next(iter_);
+    CopyValue();
+  }
+
+  const std::string &value() const override {
+    return value_;
+  }
+
+  sentencepiece::util::Status status() const override {
+    return status_;
+  }
+
+  private:
+   void CopyValue() {
+     if (item_ == nullptr) return;
+     const PyInputString ustring(item_);
+     if (ustring.IsAvalable()) {
+       const char *data = ustring.data();
+       size_t size = ustring.size();
+       while (size > 0) {
+         if (data[size - 1] == '\r' || data[size - 1] == '\n')
+           --size;
+         else
+           break;
+       }
+       value_.assign(data, size);
+     } else {
+       status_ = sentencepiece::util::Status(sentencepiece::util::StatusCode::kInternal,
+                                             "Not a string.");
+     }
+     Py_XDECREF(item_);
+   }
+   PyObject *iter_ = nullptr;
+   PyObject *item_ = nullptr;
+   std::string value_;
+   sentencepiece::util::Status status_;
+};
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+                       std::vector<int> *ids,
+                       bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+  if (!add_bos && !add_eos && !reverse) return;
+  if (reverse) std::reverse(ids->begin(), ids->end());
+  if (add_bos) ids->insert(ids->begin(), sp.bos_id());
+  if (add_eos) ids->push_back(sp.eos_id());
+}
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+                       std::vector<std::string> *pieces,
+                       bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+  if (!add_bos && !add_eos && !reverse && !emit_unk_piece) return;
+  if (reverse) std::reverse(pieces->begin(), pieces->end());
+  if (add_bos) pieces->insert(pieces->begin(), sp.IdToPiece(sp.bos_id()));
+  if (add_eos) pieces->push_back(sp.IdToPiece(sp.eos_id()));
+  if (emit_unk_piece) {
+    const auto &unk = sp.IdToPiece(sp.unk_id());
+    for (auto &piece : *pieces) {
+      const int id = sp.PieceToId(piece);
+      if (id == sp.unk_id()) {
+        piece = unk;
+      }
+    }
+  }
+}
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+                       sentencepiece::util::bytes *proto,
+                       bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+  if (add_bos || add_eos || reverse || emit_unk_piece) {
+    throw sentencepiece::util::Status(
+        sentencepiece::util::StatusCode::kUnimplemented,
+        "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API");
+  }
+}
+
+inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp,
+                       sentencepiece::ImmutableSentencePieceText *proto,
+                       bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) {
+  if (add_bos || add_eos || reverse || emit_unk_piece) {
+    throw sentencepiece::util::Status(
+        sentencepiece::util::StatusCode::kUnimplemented,
+        "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API");
+  }
+}
+
+inline void CheckIds(const std::vector<int> &ids, int num_pieces) {
+  for (int id : ids) {
+    if (id < 0 || id >= num_pieces) {
+      throw sentencepiece::util::Status(
+          sentencepiece::util::StatusCode::kOutOfRange,
+          "piece id is out of range.");
+    }
+  }
+}
+
+inline void CheckIds(const std::vector<absl::string_view> &ids, int num_pieces) {}
+
+inline void CheckIdsBatch(const std::vector<std::vector<int>> &ids, int num_pieces) {
+  for (const auto &v : ids) CheckIds(v, num_pieces);
+}
+
+template <typename T>
+inline void ConvertToUnicodeSpans(T *proto) {}
+
+template <>
+inline void ConvertToUnicodeSpans(sentencepiece::ImmutableSentencePieceText *proto) {
+  proto->ConvertToUnicodeSpans();
+}
+
+template <>
+inline void ConvertToUnicodeSpans(sentencepiece::ImmutableNBestSentencePieceText *proto) {
+  proto->ConvertToUnicodeSpans();
+}
+
+class ThreadPool {
+ public:
+  explicit ThreadPool(size_t request_size) :
+    request_size_(request_size) {}
+
+  virtual ~ThreadPool() {
+    for (auto &task : tasks_) {
+      task.join();
+    }
+  }
+
+  void Schedule(std::function<void()> closure) {
+    static constexpr size_t kMinThreadSize = 2;
+    if (request_size_ < kMinThreadSize) {
+      closure();
+    } else {
+      tasks_.emplace_back(closure);
+    }
+  }
+
+ private:
+  size_t request_size_ = 0;
+  std::vector<std::thread> tasks_;
+};
+
+template <typename T>
+inline void InitNumThreads(const std::vector<T> &ins, int *num_threads) {
+  if (*num_threads < 0) {
+    *num_threads = std::thread::hardware_concurrency();
+  }
+  *num_threads = std::max<int>(1,
+                               std::min<int>({*num_threads,
+                                     static_cast<int>(ins.size()), 256}));
+}
+
+#define DEFINE_ENCODE_BATCH_FUNC_IMPL(FuncName, InType, OutType)        \
+  std::vector<OutType> outs(ins.size());                                \
+  InitNumThreads(ins, &num_threads);                                    \
+  {                                                                     \
+    ThreadPool pool(ins.size());                                        \
+    std::atomic<size_t> index = 0;                                      \
+    for (int n = 0;  n < num_threads; ++n) {                            \
+      pool.Schedule([&]() {                                             \
+          size_t i = 0;                                                 \
+          while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \
+            auto out = enable_sampling ?                                \
+                       self->Sample##FuncName(ins[i],                   \
+                                              nbest_size, alpha) :      \
+                       self->FuncName(ins[i]);                          \
+            RewriteIds(*self, &out, add_bos, add_eos, reverse,          \
+                       emit_unk_piece);                                 \
+            ConvertToUnicodeSpans(&out);                                \
+            outs[i] = std::move(out);                                   \
+          }                                                             \
+        });                                                             \
+    }                                                                   \
+  }                                                                     \
+  return outs;
+
+#define DEFINE_DECODE_BATCH_FUNC_IMPL(FuncName, InType, OutType)        \
+  std::vector<OutType> outs(ins.size());                                \
+  InitNumThreads(ins, &num_threads);                                    \
+  {                                                                     \
+    std::atomic<size_t> index = 0;                                      \
+    ThreadPool pool(ins.size());                                        \
+    for (int n = 0;  n < num_threads; ++n) {                            \
+      pool.Schedule([&]() {                                             \
+          size_t i = 0;                                                 \
+          while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \
+            auto out = self->FuncName(ins[i]);                          \
+            ConvertToUnicodeSpans(&out);                                \
+            outs[i] = std::move(out);                                   \
+          }                                                             \
+        });                                                             \
+    }                                                                   \
+  }                                                                     \
+  return outs;
+
+}  // namespace
+%}
+
+%init %{
+#ifdef Py_GIL_DISABLED
+  PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED);
+#endif
+%}
+
+%exception {
+  try {
+    $action
+    ReleaseResultObject(resultobj);
+  }
+  catch (const sentencepiece::util::Status &status) {
+    SWIG_exception(ToSwigError(status.code()), status.ToString().c_str());
+  }
+}
+
+%apply unsigned int { uint32_t }
+
+%ignore sentencepiece::util::Status;
+%ignore sentencepiece::util::StatusCode;
+%ignore absl::string_view;
+%ignore std::string_view;
+%ignore sentencepiece::SentencePieceText;
+%ignore sentencepiece::NormalizerSpec;
+%ignore sentencepiece::TrainerSpec;
+%ignore sentencepiece::SentencePieceProcessor::status;
+%ignore sentencepiece::ImmutableSentencePieceText::mutable_proto;
+%ignore sentencepiece::ImmutableSentencePieceText::pieces() const;
+%ignore sentencepiece::ImmutableSentencePieceText::ConvertToUnicodeSpans;
+%ignore sentencepiece::ImmutableNBestSentencePieceText::mutable_proto;
+%ignore sentencepiece::ImmutableNBestSentencePieceText::nbests() const;
+%ignore sentencepiece::ImmutableNBestSentencePieceText::ConvertToUnicodeSpans;
+
+%ignore sentencepiece::SentencePieceProcessor::Encode;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncode;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncode;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScore;
+%ignore sentencepiece::SentencePieceProcessor::Decode;
+
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsIds;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsIds;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsPieces;
+%ignore sentencepiece::SentencePieceProcessor::DecodeIds;
+%ignore sentencepiece::SentencePieceProcessor::DecodePieces;
+
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsSerializedProto;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsSerializedProto;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsSerializedProto;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsSerializedProto;
+%ignore sentencepiece::SentencePieceProcessor::DecodePiecesAsSerializedProto;
+%ignore sentencepiece::SentencePieceProcessor::DecodeIdsAsSerializedProto;
+
+%ignore sentencepiece::SentencePieceProcessor::EncodeAsImmutableProto;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsImmutableProto;
+%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsImmutableProto;
+%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsImmutableProto;
+%ignore sentencepiece::SentencePieceProcessor::DecodePiecesAsImmutableProto;
+%ignore sentencepiece::SentencePieceProcessor::DecodeIdsAsImmutableProto;
+
+%ignore sentencepiece::SentencePieceProcessor::Normalize;
+%ignore sentencepiece::SentencePieceProcessor::NormalizeWithOffsets;
+
+%ignore sentencepiece::SentencePieceProcessor::model_proto;
+%ignore sentencepiece::SentencePieceProcessor::mutable_normalizer_spec;
+%ignore sentencepiece::SentencePieceProcessor::Load;
+%ignore sentencepiece::SentencePieceProcessor::LoadOrDie;
+%ignore sentencepiece::SentencePieceProcessor::SetModel;
+%ignore sentencepiece::SentencePieceProcessor::SetNormalizer;
+%ignore sentencepiece::pretokenizer::PretokenizerForTrainingInterface;
+%ignore sentencepiece::SentenceIterator;
+%ignore sentencepiece::ConvertToUnicodeSpans;
+%ignore sentencepiece::SentencePieceTrainer::Train;
+%ignore sentencepiece::SentencePieceTrainer::GetNormalizerSpec;
+%ignore sentencepiece::SentencePieceTrainer::PopulateNormalizerSpec;
+%ignore sentencepiece::SentencePieceTrainer::MergeSpecsFromArgs;
+%ignore sentencepiece::SentencePieceTrainer::SetProtoField;
+%ignore sentencepiece::SentencePieceTrainer::PopulateModelTypeFromString;
+%ignore sentencepiece::SentencePieceTrainer::PieceProcecssor;
+%ignore sentencepiece::SentencePieceTrainer::SetPretokenizerForTraining;
+%ignore sentencepiece::SentencePieceTrainer::GetPretokenizerForTraining;
+%ignore sentencepiece::SentencePieceTrainer::SetDataDir;
+%ignore sentencepiece::ConvertToUnicodeAlignment;
+
+%ignore sentencepiece::SentencePieceNormalizer::Load;
+%ignore sentencepiece::SentencePieceNormalizer::Normalize;
+%ignore sentencepiece::SentencePieceNormalizer::mutable_normalizer_spec;
+
+%ignore sentencepiece::io::LoadModelProto;
+%ignore sentencepiece::io::SaveModelProto;
+
+%extend sentencepiece::SentencePieceProcessor {
+  sentencepiece::util::Status LoadFromFile(absl::string_view arg) {
+    return $self->Load(arg);
+  }
+
+  /////////////////////////////////////////////////////////////////////////////
+  // EncodeAs* (Single request)
+  std::vector<int> _EncodeAsIds(absl::string_view text,
+                                bool enable_sampling,
+                                int nbest_size, float alpha,
+                                bool add_bos, bool add_eos, bool reverse,
+                                bool emit_unk_piece) const {
+    auto ids = enable_sampling ?
+               $self->SampleEncodeAsIds(text, nbest_size, alpha) :
+               $self->EncodeAsIds(text);
+    RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
+    return ids;
+  }
+
+  std::vector<std::string> _EncodeAsPieces(absl::string_view text,
+                                           bool enable_sampling,
+                                           int nbest_size, float alpha,
+                                           bool add_bos, bool add_eos, bool reverse,
+                                           bool emit_unk_piece) const {
+    auto pieces = enable_sampling ?
+                  $self->SampleEncodeAsPieces(text, nbest_size, alpha) :
+                  $self->EncodeAsPieces(text);
+    RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+    return pieces;
+  }
+
+  sentencepiece::util::bytes _EncodeAsSerializedProto(absl::string_view text,
+                                                      bool enable_sampling,
+                                                      int nbest_size, float alpha,
+                                                      bool add_bos, bool add_eos, bool reverse,
+                                                      bool emit_unk_piece) const {
+    auto proto = enable_sampling ?
+                 $self->SampleEncodeAsSerializedProto(text, nbest_size, alpha) :
+                 $self->EncodeAsSerializedProto(text);
+    RewriteIds(*$self, &proto, add_bos, add_eos, reverse, emit_unk_piece);
+    return proto;
+  }
+
+  sentencepiece::ImmutableSentencePieceText
+      _EncodeAsImmutableProto(absl::string_view text,
+                              bool enable_sampling,
+                              int nbest_size, float alpha,
+                              bool add_bos, bool add_eos, bool reverse,
+                              bool emit_unk_piece) const {
+    auto proto = enable_sampling ?
+                 $self->SampleEncodeAsImmutableProto(text, nbest_size, alpha) :
+                 $self->EncodeAsImmutableProto(text);
+    proto.ConvertToUnicodeSpans();
+    RewriteIds(*$self, &proto, add_bos, add_eos, reverse, emit_unk_piece);
+    return proto;
+  }
+
+  /////////////////////////////////////////////////////////////////////////////
+  // EncodeAs* (Batch request)
+  std::vector<std::vector<int>> _EncodeAsIdsBatch(
+      const std::vector<absl::string_view> &ins, int num_threads,
+      bool enable_sampling, int nbest_size, float alpha,
+      bool add_bos, bool add_eos, bool reverse,
+      bool emit_unk_piece) const {
+    DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsIds,
+                                  absl::string_view, std::vector<int>);
+  }
+
+  std::vector<std::vector<std::string>> _EncodeAsPiecesBatch(
+      const std::vector<absl::string_view> &ins, int num_threads,
+      bool enable_sampling, int nbest_size, float alpha,
+      bool add_bos, bool add_eos, bool reverse,
+      bool emit_unk_piece) const {
+    DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsPieces,
+                                  absl::string_view, std::vector<std::string>);
+  }
+
+  BytesArray _EncodeAsSerializedProtoBatch(
+      const std::vector<absl::string_view> &ins, int num_threads,
+      bool enable_sampling, int nbest_size, float alpha,
+      bool add_bos, bool add_eos, bool reverse,
+      bool emit_unk_piece) const {
+    DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsSerializedProto,
+                                  absl::string_view,
+                                  sentencepiece::util::bytes);
+  }
+
+  std::vector<sentencepiece::ImmutableSentencePieceText>
+      _EncodeAsImmutableProtoBatch(
+      const std::vector<absl::string_view> &ins, int num_threads,
+      bool enable_sampling, int nbest_size, float alpha,
+      bool add_bos, bool add_eos, bool reverse,
+      bool emit_unk_piece) const {
+    DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsImmutableProto,
+                                  absl::string_view,
+                                  sentencepiece::ImmutableSentencePieceText);
+  }
+
+  /////////////////////////////////////////////////////////////////////////////
+  // DecodeAs* (Single request)
+  std::string _DecodeIds(const std::vector<int> &ids) const {
+    CheckIds(ids, $self->GetPieceSize());
+    return $self->DecodeIds(ids);
+  }
+
+  sentencepiece::util::bytes _DecodeIdsAsBytes(const std::vector<int> &ids) const {
+    CheckIds(ids, $self->GetPieceSize());
+    return $self->DecodeIds(ids);
+  }
+
+  std::string _DecodePieces(const std::vector<absl::string_view> &pieces) const {
+    return $self->DecodePieces(pieces);
+  }
+
+  sentencepiece::util::bytes _DecodeIdsAsSerializedProto(
+      const std::vector<int> &ids) const {
+    CheckIds(ids, $self->GetPieceSize());
+    return $self->DecodeIdsAsSerializedProto(ids);
+  }
+
+  sentencepiece::util::bytes _DecodePiecesAsSerializedProto(
+      const std::vector<absl::string_view> &pieces) const {
+    CheckIds(pieces, $self->GetPieceSize());
+    return $self->DecodePiecesAsSerializedProto(pieces);
+  }
+
+  sentencepiece::ImmutableSentencePieceText _DecodeIdsAsImmutableProto(
+      const std::vector<int> &ids) const {
+    CheckIds(ids, $self->GetPieceSize());
+    auto proto = $self->DecodeIdsAsImmutableProto(ids);
+    proto.ConvertToUnicodeSpans();
+    return proto;
+  }
+
+  sentencepiece::ImmutableSentencePieceText _DecodePiecesAsImmutableProto(
+      const std::vector<absl::string_view> &pieces) const {
+    CheckIds(pieces, $self->GetPieceSize());
+    auto proto= $self->DecodePiecesAsImmutableProto(pieces);
+    proto.ConvertToUnicodeSpans();
+    return proto;
+  }
+
+  /////////////////////////////////////////////////////////////////////////////
+  // DecodeAs* (Batch request)
+  std::vector<std::string> _DecodeIdsBatch(
+      const std::vector<std::vector<int>> &ins, int num_threads) const {
+    CheckIdsBatch(ins, $self->GetPieceSize());
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string);
+  }
+
+  BytesArray _DecodeIdsAsBytesBatch(
+      const std::vector<std::vector<int>> &ins, int num_threads) const {
+    CheckIdsBatch(ins, $self->GetPieceSize());
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string);
+  }
+
+  BytesArray _DecodeIdsAsSerializedProtoBatch(
+      const std::vector<std::vector<int>> &ins, int num_threads) const {
+    CheckIdsBatch(ins, $self->GetPieceSize());
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsSerializedProto, int,
+                                  sentencepiece::util::bytes);
+  }
+
+  std::vector<sentencepiece::ImmutableSentencePieceText>
+      _DecodeIdsAsImmutableProtoBatch(
+          const std::vector<std::vector<int>> &ins, int num_threads) const {
+    CheckIdsBatch(ins, $self->GetPieceSize());
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsImmutableProto, int,
+                                  sentencepiece::ImmutableSentencePieceText);
+  }
+
+  std::vector<std::string> _DecodePiecesBatch(
+      const std::vector<std::vector<absl::string_view>> &ins, int num_threads) const {
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePieces, std::string, std::string);
+  }
+
+  BytesArray _DecodePiecesAsSerializedProtoBatch(
+      const std::vector<std::vector<absl::string_view>> &ins, int num_threads) const {
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsSerializedProto, std::string,
+                                  sentencepiece::util::bytes);
+  }
+
+  std::vector<sentencepiece::ImmutableSentencePieceText>
+      _DecodePiecesAsImmutableProtoBatch(
+          const std::vector<std::vector<absl::string_view>> &ins, int num_threads) const {
+    DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsImmutableProto, std::string,
+                                  sentencepiece::ImmutableSentencePieceText);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // NBestEncodeAs* (Single request)
+  std::vector<std::vector<int>>
+      _NBestEncodeAsIds(absl::string_view text,
+                        int nbest_size,
+                        bool add_bos, bool add_eos, bool reverse,
+                        bool emit_unk_piece) const {
+    auto idss = $self->NBestEncodeAsIds(text, nbest_size);
+    for (auto &ids : idss) {
+      RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece);
+    }
+    return idss;
+  }
+
+  std::vector<std::vector<std::string>>
+      _NBestEncodeAsPieces(absl::string_view text,
+                           int nbest_size,
+                           bool add_bos, bool add_eos, bool reverse,
+                           bool emit_unk_piece) const {
+    auto piecess = $self->NBestEncodeAsPieces(text, nbest_size);
+    for (auto &pieces : piecess) {
+      RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece);
+    }
+    return piecess;
+  }
+
+  sentencepiece::util::bytes
+      _NBestEncodeAsSerializedProto(absl::string_view text,
+                                    int nbest_size,
+                                    bool add_bos, bool add_eos, bool reverse,
+                                    bool emit_unk_piece) const {
+    RewriteIds(*$self, static_cast<sentencepiece::util::bytes *>(nullptr),
+               add_bos, add_eos, reverse, emit_unk_piece);
+    return $self->NBestEncodeAsSerializedProto(text, nbest_size);
+  }
+
+  sentencepiece::ImmutableNBestSentencePieceText
+      _NBestEncodeAsImmutableProto(absl::string_view text,
+                                   int nbest_size,
+                                   bool add_bos, bool add_eos, bool reverse,
+                                   bool emit_unk_piece) const {
+    RewriteIds(*$self, static_cast<sentencepiece::ImmutableSentencePieceText *>(nullptr),
+               add_bos, add_eos, reverse, emit_unk_piece);
+    auto proto = $self->NBestEncodeAsImmutableProto(text, nbest_size);
+    proto.ConvertToUnicodeSpans();
+    return proto;
+  }
+
+
+  /////////////////////////////////////////////////////////////////////////////
+  // SampleEncodeAndScoreAs* (Single request)
+  std::vector<std::pair<std::vector<int>, float>>
+      _SampleEncodeAndScoreAsIds(absl::string_view text,
+                                 int num_samples, float alpha, bool wor,
+                                 bool include_best,
+                                 bool add_bos, bool add_eos, bool reverse,
+                                 bool emit_unk_piece) const {
+    auto idss = $self->SampleEncodeAndScoreAsIds(text, num_samples,
+                                                 alpha, wor, include_best);
+    for (auto &ids : idss) {
+      RewriteIds(*$self, &ids.first, add_bos, add_eos, reverse, emit_unk_piece);
+    }
+    return idss;
+  }
+
+  std::vector<std::pair<std::vector<std::string>, float>>
+      _SampleEncodeAndScoreAsPieces(absl::string_view text,
+                                    int num_samples, float alpha, bool wor,
+                                    bool include_best,
+                                    bool add_bos, bool add_eos, bool reverse,
+                                    bool emit_unk_piece) const {
+    auto piecess = $self->SampleEncodeAndScoreAsPieces(text, num_samples,
+                                                       alpha, wor, include_best);
+    for (auto &pieces : piecess) {
+      RewriteIds(*$self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece);
+    }
+    return piecess;
+  }
+
+  sentencepiece::util::bytes
+      _SampleEncodeAndScoreAsSerializedProto(absl::string_view text,
+                                             int num_samples, float alpha, bool wor,
+                                             bool include_best,
+                                             bool add_bos, bool add_eos, bool reverse,
+                                             bool emit_unk_piece) const {
+    RewriteIds(*$self, static_cast<sentencepiece::util::bytes *>(nullptr),
+               add_bos, add_eos, reverse, emit_unk_piece);
+    return $self->SampleEncodeAndScoreAsSerializedProto(text, num_samples,
+                                                        alpha, wor, include_best);
+  }
+
+  sentencepiece::ImmutableNBestSentencePieceText
+      _SampleEncodeAndScoreAsImmutableProto(absl::string_view text,
+                                            int num_samples, float alpha, bool wor,
+                                            bool include_best,
+                                            bool add_bos, bool add_eos, bool reverse,
+                                            bool emit_unk_piece) const {
+    RewriteIds(*$self, static_cast<sentencepiece::util::bytes *>(nullptr),
+               add_bos, add_eos, reverse, emit_unk_piece);
+    auto proto = $self->SampleEncodeAndScoreAsImmutableProto(text, num_samples,
+                                                       alpha, wor, include_best);
+    proto.ConvertToUnicodeSpans();
+    return proto;
+  }
+
+  // Normalize
+  std::string _Normalize(absl::string_view text) {
+    return $self->Normalize(text);
+  }
+
+  std::pair<std::string, std::vector<size_t>> _NormalizeWithOffsets(absl::string_view text) {
+    std::pair<std::string, std::vector<size_t>> result;
+    $self->Normalize(text, &result.first, &result.second).IgnoreError();
+    return result;
+  }
+
+  // Calculate Entropy
+  float _CalculateEntropy(absl::string_view text, float alpha)  {
+    return $self->CalculateEntropy(text, alpha);
+  }
+
+  std::vector<float> _CalculateEntropyBatch(const std::vector<absl::string_view> &ins,
+                                            float alpha, int num_threads)  {
+    std::vector<float> outs(ins.size());
+    InitNumThreads(ins, &num_threads);
+    {
+      ThreadPool pool(ins.size());
+      std::atomic<size_t> index = 0;
+      for (int n = 0;  n < num_threads; ++n) {
+        pool.Schedule([&]() {
+           size_t i = 0;
+           while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) {
+             outs[i] = self->CalculateEntropy(ins[i], alpha);
+           }
+         });
+      }
+    }
+    return outs;
+  }
+
+  // override normalizer_spec
+  sentencepiece::util::Status _OverrideNormalizerSpec(
+      const std::unordered_map<std::string, std::string> &args) {
+    sentencepiece::util::Status status;
+    for (const auto &[key, value] : args) {
+      status = sentencepiece::SentencePieceTrainer::SetProtoField(
+          key, value,
+          $self->mutable_normalizer_spec());
+      if (!status.ok()) return status;
+    }
+    return status;
+  }
+
+%pythoncode {
+  def Init(self,
+           model_file=None,
+           model_proto=None,
+           out_type=int,
+           add_bos=False,
+           add_eos=False,
+           reverse=False,
+           emit_unk_piece=False,
+           enable_sampling=False,
+           nbest_size=-1,
+           alpha=0.1,
+           num_threads=-1):
+    """Initialzie sentencepieceProcessor.
+
+    Args:
+      model_file: The sentencepiece model file path.
+      model_proto: The sentencepiece model serialized proto.
+      out_type: output type. int or str.
+      add_bos: Add <s> to the result (Default = false)
+      add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
+        reversing (if enabled).
+      reverse: Reverses the tokenized sequence (Default = false)
+      emit_unk_piece: Emits the unk literal string (Default = false)
+      nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
+                  nbest_size = {0,1}: No sampling is performed.
+                  nbest_size > 1: samples from the nbest_size results.
+                  nbest_size < 0: assuming that nbest_size is infinite and samples
+                    from the all hypothesis (lattice) using
+                    forward-filtering-and-backward-sampling algorithm.
+      alpha: Soothing parameter for unigram sampling, and dropout probability of
+             merge operations for BPE-dropout.
+      num_threads: number of threads in batch processing (Default = -1, auto-detected)
+    """
+
+    _sentencepiece_processor_init_native(self)
+    self._out_type = out_type
+    self._add_bos = add_bos
+    self._add_eos = add_eos
+    self._reverse = reverse
+    self._emit_unk_piece = emit_unk_piece
+    self._enable_sampling = enable_sampling
+    self._nbest_size = nbest_size
+    self._alpha = alpha
+    self._num_threads = num_threads
+    if model_file or model_proto:
+      self.Load(model_file=model_file, model_proto=model_proto)
+
+
+  def Encode(self,
+             input,
+             out_type=None,
+             add_bos=None,
+             add_eos=None,
+             reverse=None,
+             emit_unk_piece=None,
+             enable_sampling=None,
+             nbest_size=None,
+             alpha=None,
+             num_threads=None):
+    """Encode text input to segmented ids or tokens.
+
+      Args:
+      input: input string. accepsts list of string.
+      out_type: output type. int or str.
+      add_bos: Add <s> to the result (Default = false)
+      add_eos: Add </s> to the result (Default = false) <s>/</s> is added after
+               reversing (if enabled).
+      reverse: Reverses the tokenized sequence (Default = false)
+      emit_unk_piece: Emits the unk literal string (Default = false)
+      nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout.
+                  nbest_size = {0,1}: No sampling is performed.
+                  nbest_size > 1: samples from the nbest_size results.
+                  nbest_size < 0: assuming that nbest_size is infinite and samples
+                  from the all hypothesis (lattice) using
+                  forward-filtering-and-backward-sampling algorithm.
+      alpha: Soothing parameter for unigram sampling, and merge probability for
+             BPE-dropout (probablity 'p' in BPE-dropout paper).
+      num_threads: the number of threads used in the batch processing (Default = -1).
+    """
+
+    if out_type is None:
+      out_type = self._out_type
+    if add_bos is None:
+      add_bos = self._add_bos
+    if add_eos is None:
+      add_eos = self._add_eos
+    if reverse is None:
+      reverse = self._reverse
+    if emit_unk_piece is None:
+      emit_unk_piece = self._emit_unk_piece
+    if enable_sampling is None:
+      enable_sampling = self._enable_sampling
+    if nbest_size is None:
+      nbest_size = self._nbest_size
+    if alpha is None:
+      alpha = self._alpha
+    if num_threads is None:
+      num_threads = self._num_threads
+
+    if enable_sampling == True and (nbest_size is None or nbest_size == 0 or
+                                    nbest_size == 1 or alpha is None):
+      raise RuntimeError(
+          'When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", '
+          'and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. '
+          'when "nbest_size = -1" , this method samples from all candidates on the lattice '
+          'instead of nbest segmentations.'
+      )
+
+    if num_threads is None or type(num_threads) is not int:
+      raise RuntimeError('num_threads must be int')
+
+    if type(input) is list:
+      if out_type is int:
+        return self._EncodeAsIdsBatch(input, num_threads, enable_sampling, nbest_size,
+                                      alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type is str:
+        return self._EncodeAsPiecesBatch(input, num_threads, enable_sampling, nbest_size,
+                                         alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'serialized_proto' or out_type == 'proto':
+        return self._EncodeAsSerializedProtoBatch(input, num_threads, enable_sampling, nbest_size,
+                                                  alpha, add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'immutable_proto':
+        return self._EncodeAsImmutableProtoBatch(input, num_threads, enable_sampling, nbest_size,
+                                                 alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    if out_type is int:
+      return self._EncodeAsIds(input, enable_sampling, nbest_size,
+                               alpha, add_bos, add_eos, reverse, emit_unk_piece)
+    if out_type is str:
+      return self._EncodeAsPieces(input, enable_sampling, nbest_size,
+                                  alpha, add_bos, add_eos, reverse, emit_unk_piece)
+    if out_type == 'serialized_proto' or out_type == 'proto':
+      return self._EncodeAsSerializedProto(input, enable_sampling, nbest_size,
+                                           alpha, add_bos, add_eos, reverse, emit_unk_piece)
+    if out_type == 'immutable_proto':
+      return self._EncodeAsImmutableProto(input, enable_sampling, nbest_size,
+                                          alpha, add_bos, add_eos, reverse, emit_unk_piece)
+
+    raise RuntimeError('unknown out_type={}'.format(out_type))
+    return None
+
+
+  def EncodeAsPieces(self, input, **kwargs):
+    return self.Encode(input=input, out_type=str, **kwargs)
+
+
+  def EncodeAsIds(self, input, **kwargs):
+    return self.Encode(input=input, out_type=int, **kwargs)
+
+
+  def EncodeAsSerializedProto(self, input, **kwargs):
+    return self.Encode(input=input, out_type='serialized_proto', **kwargs)
+
+
+  def EncodeAsImmutableProto(self, input, **kwargs):
+    return self.Encode(input=input, out_type='immutable_proto', **kwargs)
+
+
+  def SampleEncodeAsPieces(self, input, nbest_size=None, alpha=None, **kwargs):
+    return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                       out_type=str, enable_sampling=True, **kwargs)
+
+
+  def SampleEncodeAsIds(self, input, nbest_size=None, alpha=None,**kwargs):
+    return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                       out_type=int, enable_sampling=True, **kwargs)
+
+
+  def SampleEncodeAsSerializedProto(self, input, nbest_size=None, alpha=None, **kwargs):
+    return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                       out_type='serialized_proto', enable_sampling=True, **kwargs)
+
+
+  def SampleEncodeAsImmutableProto(self, input, nbest_size=None, alpha=None, **kwargs):
+    return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha,
+                       out_type='immutable_proto', enable_sampling=True, **kwargs)
+
+
+  def NBestEncode(self,
+                  input,
+                  out_type=None,
+                  add_bos=None,
+                  add_eos=None,
+                  reverse=None,
+                  emit_unk_piece=None,
+                  nbest_size=None):
+    """NBestEncode text input to segmented ids or tokens.
+
+      Args:
+      input: input string. accepsts list of string.
+      out_type: output type. int or str.
+      add_bos: Add <s> to the result (Default = false)
+      add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
+      reverse: Reverses the tokenized sequence (Default = false)
+      emit_unk_piece: Emits the unk literal string (Default = false)
+      nbest_size: nbest size
+    """
+
+    if out_type is None:
+      out_type = self._out_type
+    if add_bos is None:
+      add_bos = self._add_bos
+    if add_eos is None:
+      add_eos = self._add_eos
+    if reverse is None:
+      reverse = self._reverse
+    if emit_unk_piece is None:
+      emit_unk_piece = self._emit_unk_piece
+    if nbest_size is None:
+      nbest_size = self._nbest_size
+
+    if nbest_size <= 0:
+      nbest_size=1
+
+    def _encode(text):
+      if out_type is int:
+        return self._NBestEncodeAsIds(text, nbest_size,
+                                      add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type is str:
+        return self._NBestEncodeAsPieces(text, nbest_size,
+                                         add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'serialized_proto' or out_type == 'proto':
+        return self._NBestEncodeAsSerializedProto(text, nbest_size,
+                                                  add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type == 'immutable_proto':
+        return self._NBestEncodeAsImmutableProto(text, nbest_size,
+                                                 add_bos, add_eos, reverse, emit_unk_piece)
+
+      raise RuntimeError('unknown out_type')
+
+    if type(input) is list:
+      return [_encode(n) for n in input]
+
+    return _encode(input)
+
+
+  def NBestEncodeAsPieces(self, input, nbest_size=None, **kwargs):
+    return self.NBestEncode(input=input, nbest_size=nbest_size,
+                            out_type=str, **kwargs)
+
+
+  def NBestEncodeAsIds(self, input, nbest_size=None, **kwargs):
+    return self.NBestEncode(input=input, nbest_size=nbest_size,
+                            out_type=int, **kwargs)
+
+
+  def NBestEncodeAsSerializedProto(self, input, nbest_size=None, **kwargs):
+    return self.NBestEncode(input=input, nbest_size=nbest_size,
+                            out_type='serialized_proto', **kwargs)
+
+
+  def NBestEncodeAsImmutableProto(self, input, nbest_size=None, **kwargs):
+    return self.NBestEncode(input=input, nbest_size=nbest_size,
+                            out_type='immutable_proto', **kwargs)
+
+
+  def SampleEncodeAndScore(self,
+                           input,
+                           out_type=None,
+                           add_bos=None,
+                           add_eos=None,
+                           reverse=None,
+                           emit_unk_piece=None,
+                           num_samples=None,
+                           alpha=None,
+                           wor=None,
+                           include_best=None):
+    """SampleEncodeAndScore text input to segmented ids or tokens.
+
+      Args:
+      input: input string. accepsts list of string.
+      out_type: output type. int or str or 'serialized_proto' or 'immutable_proto'
+      add_bos: Add <s> to the result (Default = false)
+      add_eos: Add </s> to the result (Default = false) <s>/</s> is added after reversing (if enabled).
+      reverse: Reverses the tokenized sequence (Default = false)
+      emit_unk_piece: Emits the unk literal string (Default = false)
+      num_samples: How many samples to return (Default = 1)
+      alpha: inverse temperature for sampling
+      wor: whether to sample without replacement (Default = false)
+      include_best: whether to include the best tokenization, requires wor=True (Default = false)
+    """
+
+    if out_type is None:
+      out_type = self._out_type
+    if add_bos is None:
+      add_bos = self._add_bos
+    if add_eos is None:
+      add_eos = self._add_eos
+    if reverse is None:
+      reverse = self._reverse
+    if emit_unk_piece is None:
+      emit_unk_piece = self._emit_unk_piece
+    if num_samples is None:
+      num_samples = 1
+    if alpha is None:
+      alpha = 1.
+    if wor is None:
+      wor = False
+    if include_best is None:
+      include_best = False
+
+    if num_samples <= 0:
+      raise RuntimeError('num_examples must be positive')
+
+    if include_best and not wor:
+      raise RuntimeError('When include_best is True, We must specify "wor = True".')
+
+
+    def _encode(text):
+      if out_type is int:
+        return self._SampleEncodeAndScoreAsIds(text, num_samples, alpha, wor, include_best,
+                                               add_bos, add_eos, reverse, emit_unk_piece)
+      if out_type is str:
+        return self._SampleEncodeAndScoreAsPieces(text, num_samples, alpha, wor, include_best,
+                                                  add_bos, add_eos, reverse, emit_unk_piece)
+
+      if out_type == 'serialized_proto' or out_type == 'proto':
+        return self._SampleEncodeAndScoreAsSerializedProto(text, num_samples, alpha, wor, include_best,
+                                                           add_bos, add_eos, reverse, emit_unk_piece)
+
+      if out_type == 'immutable_proto':
+        return self._SampleEncodeAndScoreAsImmutableProto(text, num_samples, alpha, wor, include_best,
+                                                          add_bos, add_eos, reverse, emit_unk_piece)
+
+      raise RuntimeError('unknown output type')
+
+
+    if type(input) is list:
+      return [_encode(n) for n in input]
+
+    return _encode(input)
+
+
+  def SampleEncodeAndScoreAsPieces(self, input, num_samples=None, alpha=None, **kwargs):
+    return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                     out_type=str, **kwargs)
+
+
+  def SampleEncodeAndScoreAsIds(self, input, num_samples=None, alpha=None, **kwargs):
+    return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                     out_type=int, **kwargs)
+
+
+  def SampleEncodeAndScoreAsSerializedProto(self, input, num_samples=None, alpha=None, **kwargs):
+    return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                     out_type='serialized_proto', **kwargs)
+
+
+  def SampleEncodeAndScoreAsImmutableProto(self, input, num_samples=None, alpha=None, **kwargs):
+    return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha,
+                                     out_type='immutable_proto', **kwargs)
+
+
+  def Decode(self, input, out_type=str, num_threads=None):
+    """Decode processed id or token sequences.
+
+    Args:
+      out_type: output type. str, bytes or 'serialized_proto' or 'immutable_proto' (Default = str)
+      num_threads: the number of threads used in the batch processing (Default = -1).
+    """
+
+    if num_threads is None:
+      num_threads = self._num_threads
+
+    if num_threads is None or type(num_threads) is not int:
+      raise RuntimeError('num_threads must be int')
+
+    if not input:
+      return ''
+
+    if out_type is str:
+      if type(input) is int:
+        return self._DecodeIds([input])
+      if type(input) is str:
+        return self._DecodePieces([input])
+
+      if type(input) is list:
+        if len(input) == 0 or type(input[0]) is int:
+          return self._DecodeIds(input)
+        if type(input[0]) is str:
+          return self._DecodePieces(input)
+
+        if type(input[0]) is list:
+          if len(input[0]) == 0 or type(input[0][0]) is int:
+           return self._DecodeIdsBatch(input, num_threads)
+          if type(input[0][0]) is str:
+           return self._DecodePiecesBatch(input, num_threads)
+
+    if out_type is bytes:
+      if type(input) is int:
+        return self._DecodeIdsAsBytes([input])
+      if type(input) is str:
+        return self._DecodePieces([input])
+
+      if type(input) is list:
+        if len(input) == 0 or type(input[0]) is int:
+          return self._DecodeIdsAsBytes(input)
+        if type(input[0]) is str:
+          return self._DecodePieces(input)
+
+        if type(input[0]) is list:
+          if len(input[0]) == 0 or type(input[0][0]) is int:
+           return self._DecodeIdsAsBytesBatch(input, num_threads)
+          if type(input[0][0]) is str:
+           return self._DecodePiecesBatch(input, num_threads)
+
+    if out_type == 'serialized_proto':
+      if type(input) is int:
+        return self._DecodeIdsAsSerializedProto([input])
+      if type(input) is str:
+        return self._DecodePiecesAsSerializedProto([input])
+
+      if type(input) is list:
+        if len(input) == 0 or type(input[0]) is int:
+          return self._DecodeIdsAsSerializedProto(input)
+        if type(input[0]) is str:
+          return self._DecodePiecesAsSerializedProto(input)
+
+        if type(input[0]) is list:
+          if len(input[0]) == 0 or type(input[0][0]) is int:
+           return self._DecodeIdsAsSerializedProtoBatch(input, num_threads)
+          if type(input[0][0]) is str:
+           return self._DecodePiecesAsSerializedProtoBatch(input, num_threads)
+
+
+    if out_type == 'immutable_proto':
+      if type(input) is int:
+        return self._DecodeIdsAsImmutableProto([input])
+      if type(input) is str:
+        return self._DecodePiecesAsImmutableProto([input])
+
+      if type(input) is list:
+        if len(input) == 0 or type(input[0]) is int:
+          return self._DecodeIdsAsImmutableProto(input)
+        if type(input[0]) is str:
+          return self._DecodePiecesAsImmutableProto(input)
+
+        if type(input[0]) is list:
+          if len(input[0]) == 0 or type(input[0][0]) is int:
+           return self._DecodeIdsAsImmutableProtoBatch(input, num_threads)
+          if type(input[0][0]) is str:
+           return self._DecodePiecesAsImmutableProtoBatch(input, num_threads)
+
+
+    raise RuntimeError('unknown output or input type')
+    return None
+
+
+  def DecodePieces(self, input, out_type=str, **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def DecodeIds(self, input, out_type=str, **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def DecodePiecesAsSerializedProto(self, input, out_type='serialized_proto', **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def DecodeIdsAsSerializedProto(self, input, out_type='serialized_proto', **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def DecodePiecesAsImmutableProto(self, input, out_type='immutable_proto', **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def DecodeIdsAsImmutableProto(self, input, out_type='immutable_proto', **kwargs):
+    return self.Decode(input=input, out_type=out_type, **kwargs)
+
+
+  def CalculateEntropy(self, input, alpha, num_threads=None):
+    """Calculate sentence entropy"""
+    if type(input) is list:
+      if num_threads is None:
+        num_threads = self._num_threads
+      if num_threads is None or type(num_threads) is not int:
+        raise RuntimeError('num_threads must be int')
+      return self._CalculateEntropyBatch(input, alpha, num_threads)
+
+    return self._CalculateEntropy(input, alpha)
+
+
+  def Normalize(self, input, with_offsets=None):
+    def _normalize(text):
+      if with_offsets:
+        return self._NormalizeWithOffsets(text)
+      return self._Normalize(text)
+
+    if type(input) is list:
+      return [_normalize(x) for x in input]
+    return _normalize(input)
+
+  def OverrideNormalizerSpec(self, **kwargs):
+    new_kwargs = {}
+    for key, value in kwargs.items():
+      new_kwargs[key] = str(value)
+    return self._OverrideNormalizerSpec(new_kwargs)
+
+
+  def piece_size(self):
+    return self.GetPieceSize()
+
+
+  def vocab_size(self):
+    return self.GetPieceSize()
+
+
+  def __getstate__(self):
+    return self.serialized_model_proto()
+
+
+  def __setstate__(self, serialized_model_proto):
+    self.__init__()
+    self.LoadFromSerializedProto(serialized_model_proto)
+
+
+  def __len__(self):
+    return self.GetPieceSize()
+
+
+  def __getitem__(self, piece):
+    return self.PieceToId(piece)
+
+
+  def Load(self, model_file=None, model_proto=None):
+    """Overwride SentencePieceProcessor.Load to support both model_file and model_proto.
+
+    Args:
+      model_file: The sentencepiece model file path.
+      model_proto: The sentencepiece model serialized proto. Either `model_file`
+        or `model_proto` must be set.
+    """
+    if model_file and model_proto:
+      raise RuntimeError('model_file and model_proto must be exclusive.')
+    if model_proto:
+      return self.LoadFromSerializedProto(model_proto)
+    return self.LoadFromFile(model_file)
+}
+}
+
+%extend sentencepiece::SentencePieceTrainer {
+  static void _TrainFromString(absl::string_view arg) {
+    const auto _status = sentencepiece::SentencePieceTrainer::Train(arg);
+    if (!_status.ok()) throw _status;
+    return;
+  }
+
+  static void _TrainFromMap(const std::unordered_map<std::string, std::string> &args) {
+    const auto _status = sentencepiece::SentencePieceTrainer::Train(args);
+    if (!_status.ok()) throw _status;
+    return;
+  }
+
+  static void _TrainFromMap2(const std::unordered_map<std::string, std::string> &args,
+                            SentenceIterator *iter) {
+    const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter);
+    if (!_status.ok()) throw _status;
+    return;
+  }
+
+  static sentencepiece::util::bytes _TrainFromMap3(const std::unordered_map<std::string, std::string> &args) {
+    sentencepiece::util::bytes model_proto;
+    const auto _status = sentencepiece::SentencePieceTrainer::Train(args, nullptr, &model_proto);
+    if (!_status.ok()) throw _status;
+    return model_proto;
+  }
+
+  static sentencepiece::util::bytes _TrainFromMap4(const std::unordered_map<std::string, std::string> &args,
+                                                  SentenceIterator *iter) {
+    sentencepiece::util::bytes model_proto;
+    const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter, &model_proto);
+    if (!_status.ok()) throw _status;
+    return model_proto;
+  }
+
+%pythoncode {
+  @staticmethod
+  def _Train(arg=None, **kwargs):
+    """Train Sentencepiece model. Accept both kwargs and legacy string arg."""
+    if arg is not None and type(arg) is str:
+      return SentencePieceTrainer._TrainFromString(arg)
+
+    def _encode(value):
+      """Encode value to CSV.."""
+      if type(value) is list:
+        if sys.version_info[0] == 3:
+          f = StringIO()
+        else:
+          f = BytesIO()
+        writer = csv.writer(f, lineterminator='')
+        writer.writerow([str(v) for v in value])
+        return f.getvalue()
+      else:
+        return str(value)
+
+    sentence_iterator = None
+    model_writer = None
+    new_kwargs = {}
+    for key, value in kwargs.items():
+      if key in ['sentence_iterator', 'sentence_reader']:
+        sentence_iterator = value
+      elif key in ['model_writer']:
+        model_writer = value
+      else:
+        new_kwargs[key] = _encode(value)
+
+    if model_writer:
+      if sentence_iterator:
+        model_proto = SentencePieceTrainer._TrainFromMap4(new_kwargs,
+                                                         sentence_iterator)
+      else:
+        model_proto = SentencePieceTrainer._TrainFromMap3(new_kwargs)
+      model_writer.write(model_proto)
+    else:
+      if sentence_iterator:
+        return SentencePieceTrainer._TrainFromMap2(new_kwargs, sentence_iterator)
+      else:
+        return SentencePieceTrainer._TrainFromMap(new_kwargs)
+
+    return None
+
+  @staticmethod
+  def Train(arg=None, logstream=None, **kwargs):
+    with _LogStream(ostream=logstream):
+      SentencePieceTrainer._Train(arg=arg, **kwargs)
+}
+}
+
+%extend sentencepiece::SentencePieceNormalizer {
+  sentencepiece::util::Status LoadFromFile(absl::string_view arg) {
+    return $self->Load(arg);
+  }
+
+  std::string _Normalize(absl::string_view text) {
+    std::string result;
+    const auto _status = $self->Normalize(text, &result);
+    if (!_status.ok()) throw _status;
+    return result;
+  }
+
+  std::pair<std::string, std::vector<size_t>> _NormalizeWithOffsets(absl::string_view text) {
+    std::pair<std::string, std::vector<size_t>> result;
+    const auto _status = $self->Normalize(text, &result.first, &result.second);
+    if (!_status.ok()) throw _status;
+    return result;
+  }
+
+  void _SetProtoField(absl::string_view name, bool value) {
+    sentencepiece::SentencePieceTrainer::SetProtoField(
+        name,
+        value ? "1" : "0",
+        $self->mutable_normalizer_spec()).IgnoreError();
+  }
+
+%pythoncode %{
+  def Init(self,
+           model_file=None,
+           model_proto=None,
+           rule_tsv=None,
+           rule_name=None,
+           add_dummy_prefix=False,
+           escape_whitespaces=False,
+           remove_extra_whitespaces=False):
+    """Initialzie sentencePieceNormalizer.
+
+    Args:
+      model_file: The sentencepiece model file path.
+      model_proto: The sentencepiece model serialized proto.
+      rule_tsv: The normalization rule file in TSV format.
+      rule_name: Pre-defined normalization name.
+      add_dummy_prefix: add dummy prefix.
+      escape_whitespaces: escape whitespaces.
+      remove_extra_whitespaces: remove extra whitespaces.
+    """
+
+    _sentencepiece_normalizer_init_native(self)
+
+    if model_file:
+      status = self.LoadFromFile(model_file)
+    elif model_proto:
+      status = self.LoadFromSerializedProto(model_proto)
+    elif rule_tsv:
+      status = self.LoadFromRuleTSV(rule_tsv)
+    elif rule_name:
+      status = self.LoadFromRuleName(rule_name)
+    else:
+      raise RuntimeError('no model is specified')
+
+    if status:
+      self._SetProtoField('add_dummy_prefix', add_dummy_prefix)
+      self._SetProtoField('escape_whitespaces', escape_whitespaces)
+      self._SetProtoField('remove_extra_whitespaces', remove_extra_whitespaces)
+
+  def Normalize(self, input, with_offsets=None):
+    def _normalize(text):
+      if with_offsets:
+        return self._NormalizeWithOffsets(text)
+      return self._Normalize(text)
+
+    if type(input) is list:
+      return [_normalize(x) for x in input]
+    return _normalize(input)
+
+
+  def __getstate__(self):
+    return self.serialized_model_proto()
+
+
+  def __setstate__(self, serialized_model_proto):
+    self.__init__()
+    self.LoadFromSerializedProto(serialized_model_proto)
+%}
+}
+
+%extend sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece {
+  const sentencepiece::util::bytes& _surface_as_bytes() const {
+    return $self->surface();
+  }
+
+  const sentencepiece::util::bytes& _piece_as_bytes() const {
+    return $self->piece();
+  }
+
+  %rename(_piece) piece;
+  %rename(_piece_as_bytes) piece_as_bytes;
+  %rename(_id) id;
+  %rename(_surface) surface;
+  %rename(_surface_as_bytes) surface_as_bytes;
+  %rename(_begin) begin;
+  %rename(_end) end;
+
+  %pythoncode %{
+    piece = property(_piece)
+    piece_as_bytes = property(_piece_as_bytes)
+    surface = property(_surface)
+    surface_as_bytes = property(_surface_as_bytes)
+    id = property(_id)
+    begin = property(_begin)
+    end = property(_end)
+
+    def __str__(self):
+      return ('piece: \"{}\"\n'
+              'id: {}\n'
+              'surface: \"{}\"\n'
+              'begin: {}\n'
+              'end: {}\n').format(self.piece, self.id, self.surface,
+                                  self.begin, self.end)
+
+    def __eq__(self, other):
+      return self.piece == other.piece and self.id == other.id and self.surface == other.surface and self.begin == other.begin and self.end == other.end
+
+    def __hash__(self):
+      return hash(str(self))
+
+    __repr__ = __str__
+  %}
+}
+
+%extend sentencepiece::ImmutableSentencePieceText {
+  const sentencepiece::util::bytes& _text_as_bytes() const {
+    return $self->text();
+  }
+
+  %rename(_text) text;
+  %rename(_text_as_bytes) text_as_bytes;
+  %rename(_score) score;
+  %rename(_pieces) pieces;
+  %rename(_pieces_size) pieces_size;
+
+  %pythoncode %{
+    text = property(_text)
+    text_as_bytes = property(_text_as_bytes)
+    score = property(_score)
+
+    class ImmutableSentencePieceIterator:
+      def __init__(self, proto):
+        self.proto = proto
+        self.len = self.proto._pieces_size()
+
+      def __len__(self):
+        return self.len
+
+      def __getitem__(self, index):
+        if isinstance(index, slice):
+          return [self.proto._pieces(i) for i in range(self.len)][index.start:index.stop:index.step]
+        if index < 0:
+          index = index + self.len
+        if index < 0 or index >= self.len:
+          raise IndexError('piece index is out of range')
+        return self.proto._pieces(index)
+
+      def __str__(self):
+        return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self])
+
+      __repr__ = __str__
+
+    @property
+    def pieces(self):
+      return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self)
+
+    def __eq__(self, other):
+      return self.SerializeAsString() == other.SerializeAsString()
+
+    def __hash__(self):
+      return hash(self.SerializeAsString())
+
+    def __str__(self):
+      return ('text: \"{}\"\n'
+              'score: {}\n'
+              '{}').format(self.text, self.score,
+                           '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces]))
+
+    __repr__ = __str__
+  %}
+}
+
+%extend sentencepiece::ImmutableNBestSentencePieceText {
+  %rename(_nbests) nbests;
+  %rename(_nbests_size) nbests_size;
+
+  %pythoncode %{
+    class ImmutableSentencePieceTextIterator:
+      def __init__(self, proto):
+        self.proto = proto
+        self.len = self.proto._nbests_size()
+
+      def __len__(self):
+        return self.len
+
+      def __getitem__(self, index):
+        if isinstance(index, slice):
+          return [self.proto._nbests(i) for i in range(self.len)][index.start:index.stop:index.step]
+        if index < 0:
+          index = index + self.len
+        if index < 0 or index >= self.len:
+          raise IndexError('nbests index is out of range')
+        return self.proto._nbests(index)
+
+      def __str__(self):
+        return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self])
+
+      __repr__ = __str__
+
+    @property
+    def nbests(self):
+      return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self)
+
+    def __eq__(self, other):
+      return self.SerializeAsString() == other.SerializeAsString()
+
+    def __hash__(self):
+      return hash(self.SerializeAsString())
+
+    def __str__(self):
+      return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests])
+
+    __repr__ = __str__
+  %}
+}
+
+%typemap(out) std::vector<int> {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyList_SET_ITEM($result, i, PyInt_FromLong(static_cast<long>($1[i])));
+  }
+}
+
+%typemap(out) std::vector<float> {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyList_SET_ITEM($result, i, PyFloat_FromDouble(static_cast<double>($1[i])));
+  }
+}
+
+%typemap(out) std::vector<std::vector<int>> {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyObject *obj = PyList_New($1[i].size());
+    for (size_t j = 0; j < $1[i].size(); ++j) {
+      PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast<long>($1[i][j])));
+    }
+    PyList_SET_ITEM($result, i, obj);
+  }
+}
+
+%typemap(out) std::vector<std::string> {
+  PyObject *input_type = resultobj;
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyList_SET_ITEM($result, i, MakePyOutputString($1[i], input_type));
+  }
+}
+
+%typemap(out) BytesArray {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyList_SET_ITEM($result, i, MakePyOutputBytes($1[i]));
+  }
+}
+
+%typemap(out) std::vector<std::vector<std::string>> {
+  PyObject *input_type = resultobj;
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyObject *obj = PyList_New($1[i].size());
+    for (size_t j = 0; j < $1[i].size(); ++j) {
+      PyList_SET_ITEM(obj, j, MakePyOutputString($1[i][j], input_type));
+    }
+    PyList_SET_ITEM($result, i, obj);
+  }
+}
+
+%typemap(out) sentencepiece::util::bytes {
+  $result = MakePyOutputBytes($1);
+}
+
+%typemap(out) const sentencepiece::util::bytes& {
+  $result = MakePyOutputBytes(*$1);
+}
+
+%typemap(out) std::string {
+  PyObject *input_type = resultobj;
+  $result = MakePyOutputString($1, input_type);
+}
+
+%typemap(out) const std::string& {
+  PyObject *input_type = resultobj;
+  $result = MakePyOutputString(*$1, input_type);
+}
+
+%typemap(out) sentencepiece::util::Status {
+  if (!$1.ok()) {
+    SWIG_exception(ToSwigError($1.code()), $1.ToString().c_str());
+  }
+  $result = SWIG_From_bool($1.ok());}
+
+
+%typemap(in) const std::string & {
+  const PyInputString ustring($input);
+  if (!ustring.IsAvalable()) {
+    PyErr_SetString(PyExc_TypeError, "not a string");
+    SWIG_fail;
+  }
+  resultobj = ustring.input_type();
+  $1 = new std::string(ustring.data(), ustring.size());
+}
+
+%typemap(typecheck) absl::string_view = char *;
+
+%typemap(in) absl::string_view {
+  const PyInputString ustring($input);
+  if (!ustring.IsAvalable()) {
+    PyErr_SetString(PyExc_TypeError, "not a string");
+    SWIG_fail;
+  }
+  resultobj = ustring.input_type();
+  $1 = ustring.str();
+}
+
+%typemap(in) const std::vector<absl::string_view>& {
+  std::vector<absl::string_view> *out = nullptr;
+  if (PyList_Check($input)) {
+    const size_t size = PyList_Size($input);
+    out = new std::vector<absl::string_view>(size);
+    for (size_t i = 0; i < size; ++i) {
+      const PyInputString ustring(PyList_GetItem($input, i));
+      if (ustring.IsAvalable()) {
+        (*out)[i] = ustring.str();
+      } else {
+        PyErr_SetString(PyExc_TypeError, "list must contain strings");
+        SWIG_fail;
+      }
+      resultobj = ustring.input_type();
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError, "not a list");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(in) const std::vector<int>& {
+  std::vector<int> *out = nullptr;
+  if (PyList_Check($input)) {
+    const size_t size = PyList_Size($input);
+    out = new std::vector<int>(size);
+    for (size_t i = 0; i < size; ++i) {
+      PyObject *o = PyList_GetItem($input, i);
+      if (PyInt_Check(o)) {
+        (*out)[i] = static_cast<int>(PyInt_AsLong(o));
+      } else {
+        PyErr_SetString(PyExc_TypeError,"list must contain integers");
+        SWIG_fail;
+      }
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError,"not a list");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(in) const std::vector<std::vector<absl::string_view>>& {
+  std::vector<std::vector<absl::string_view>> *out = nullptr;
+  if (PyList_Check($input)) {
+    const size_t size = PyList_Size($input);
+    out = new std::vector<std::vector<absl::string_view>>(size);
+    for (size_t i = 0; i < size; ++i) {
+      PyObject *o = PyList_GetItem($input, i);
+      if (PyList_Check(o)) {
+        const size_t size2 = PyList_Size(o);
+        (*out)[i].resize(size2);
+        for (size_t j = 0; j < size2; ++j) {
+          const PyInputString ustring(PyList_GetItem(o, j));
+          if (ustring.IsAvalable()) {
+            (*out)[i][j] = ustring.str();
+          } else {
+            PyErr_SetString(PyExc_TypeError,"list must contain integers");
+            SWIG_fail;
+          }
+          resultobj = ustring.input_type();
+        }
+      } else {
+        PyErr_SetString(PyExc_TypeError,"not a list");
+        SWIG_fail;
+      }
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError,"not a list");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(in) const std::vector<std::vector<int>>& {
+  std::vector<std::vector<int>> *out = nullptr;
+  if (PyList_Check($input)) {
+    const size_t size = PyList_Size($input);
+    out = new std::vector<std::vector<int>>(size);
+    for (size_t i = 0; i < size; ++i) {
+      PyObject *o = PyList_GetItem($input, i);
+      if (PyList_Check(o)) {
+        const size_t size2 = PyList_Size(o);
+        (*out)[i].resize(size2);
+        for (size_t j = 0; j < size2; ++j) {
+          PyObject *o2 = PyList_GetItem(o, j);
+          if (PyInt_Check(o2)) {
+            (*out)[i][j] = static_cast<int>(PyInt_AsLong(o2));
+          } else {
+            PyErr_SetString(PyExc_TypeError, "list must contain strings");
+            SWIG_fail;
+          }
+        }
+      } else {
+        PyErr_SetString(PyExc_TypeError, "not a list");
+        SWIG_fail;
+      }
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError,"not a list");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(in) const std::unordered_map<std::string, std::string> & {
+  std::unordered_map<std::string, std::string> *out = nullptr;
+  if (PyDict_Check($input)) {
+    PyObject *key, *value;
+    Py_ssize_t pos = 0;
+    out = new std::unordered_map<std::string, std::string>;
+    while (PyDict_Next($input, &pos, &key, &value)) {
+      const PyInputString key_ustring(key);
+      const PyInputString value_ustring(value);
+      if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) {
+        out->emplace(std::string(key_ustring.data(), key_ustring.size()),
+                     std::string(value_ustring.data(), value_ustring.size()));
+      } else {
+        PyErr_SetString(PyExc_TypeError, "map must contain strings.");
+        SWIG_fail;
+      }
+      resultobj = key_ustring.input_type();
+    }
+  } else {
+    PyErr_SetString(PyExc_TypeError, "not a dictionary");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(out) std::vector<std::pair<std::vector<std::string>, float>> {
+  PyObject *input_type = resultobj;
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyObject *obj = PyList_New($1[i].first.size());
+    for (size_t j = 0; j < $1[i].first.size(); ++j) {
+      PyList_SET_ITEM(obj, j, MakePyOutputString($1[i].first[j], input_type));
+    }
+    PyList_SET_ITEM($result, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>($1[i].second))));
+  }
+}
+
+%typemap(out) std::vector<std::pair<std::vector<int>, float>> {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyObject *obj = PyList_New($1[i].first.size());
+    for (size_t j = 0; j < $1[i].first.size(); ++j) {
+      PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast<long>($1[i].first[j])));
+    }
+    PyList_SET_ITEM($result, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast<double>($1[i].second))));
+  }
+}
+
+%typemap(out) std::vector<sentencepiece::ImmutableSentencePieceText> {
+  $result = PyList_New($1.size());
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyObject *obj = SWIG_NewPointerObj(new sentencepiece::ImmutableSentencePieceText($1.at(i)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0);
+    PyList_SET_ITEM($result, i, obj);
+  }
+}
+
+// Types for normalized string and offset
+%typemap(out) std::pair<std::string, std::vector<size_t>> {
+  PyObject *input_type = resultobj;
+  if (PyInputString::IsUnicode(input_type)) {
+    sentencepiece::ConvertToUnicodeAlignment(arg2, $1.first, &$1.second);
+  }
+  PyObject *obj = PyList_New($1.second.size());
+  for (size_t i = 0; i < $1.second.size(); ++i) {
+    PyList_SET_ITEM(obj, i, PyInt_FromLong(static_cast<long>($1.second[i])));
+  }
+  $result = PyTuple_Pack(2, MakePyOutputString($1.first, input_type), obj);
+}
+
+%typemap(in) sentencepiece::SentenceIterator * {
+  sentencepiece::SentenceIterator *out = nullptr;
+  if (PyIter_Check($input)) {
+    out = new PySentenceIterator($input);
+  } else {
+    PyErr_SetString(PyExc_TypeError, "not a iterator");
+    SWIG_fail;
+  }
+  $1 = out;
+}
+
+%typemap(freearg) const std::string& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<std::string>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<absl::string_view>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<std::vector<std::string>>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<int>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<float>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::vector<std::vector<int>>& {
+  delete $1;
+}
+
+%typemap(freearg) const std::unordered_map<std::string, std::string> & {
+  delete $1;
+}
+
+%typemap(freearg) sentencepiece::SentenceIterator * {
+  delete $1;
+}
+
+%typemap(freearg) sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece {
+  delete $1;
+}
+
+%typemap(freearg) sentencepiece::ImmutableSentencePieceText {
+  delete $1;
+}
+
+%typemap(freearg) sentencepiece::ImmutableNBestSentencePieceText {
+  delete $1;
+}
+
+%include <sentencepiece_processor.h>
+%include <sentencepiece_trainer.h>
+
+%pythoncode %{
+
+import re
+import csv
+import sys
+import os
+import importlib.resources
+from io import StringIO
+from io import BytesIO
+
+
+def _add_snake_case(classname):
+  """Added snake_cased method from CammelCased method."""
+
+  snake_map = {}
+  for k, v in classname.__dict__.items():
+    if re.match(r'^[A-Z]+', k):
+      snake = re.sub(r'(?<!^)(?=[A-Z])', '_',
+                     k).lower().replace('n_best', 'nbest')
+      snake_map[snake] = v
+  for k, v in snake_map.items():
+    setattr(classname, k, v)
+
+
+def _batchnize(classname, name):
+  """Enables batch request for the method classname.name."""
+  func = getattr(classname, name, None)
+  def _func(v, n):
+    if type(n) is int and (n < 0 or n >= v.piece_size()):
+      raise IndexError('piece id is out of range.')
+    return func(v, n)
+
+  def _batched_func(self, arg):
+    if type(arg) is list:
+      return [_func(self, n) for n in arg]
+    else:
+      return _func(self, arg)
+
+  setattr(classname, name, _batched_func)
+
+
+_sentencepiece_processor_init_native = SentencePieceProcessor.__init__
+_sentencepiece_normalizer_init_native = SentencePieceNormalizer.__init__
+setattr(SentencePieceProcessor, '__init__', SentencePieceProcessor.Init)
+setattr(SentencePieceNormalizer, '__init__', SentencePieceNormalizer.Init)
+
+SentencePieceProcessor.Tokenize = SentencePieceProcessor.Encode
+SentencePieceProcessor.Detokenize = SentencePieceProcessor.Decode
+
+for m in [
+    'PieceToId', 'IdToPiece', 'GetScore', 'IsUnknown', 'IsControl', 'IsUnused',
+    'IsByte'
+]:
+  _batchnize(SentencePieceProcessor, m)
+
+_add_snake_case(SentencePieceProcessor)
+_add_snake_case(SentencePieceTrainer)
+_add_snake_case(SentencePieceNormalizer)
+set_random_generator_seed = SetRandomGeneratorSeed
+set_min_log_level = SetMinLogLevel
+
+from ._version import __version__
+
+SetDataDir(os.path.join(str(importlib.resources.files('sentencepiece')), 'package_data'))
+
+class _LogStream(object):
+  def __init__(self, ostream=None):
+    self.ostream = ostream
+    if self.ostream is not None:
+      self.orig_stream_fileno = sys.stderr.fileno()
+
+  def __enter__(self):
+    if self.ostream is not None:
+      self.orig_stream_dup = os.dup(self.orig_stream_fileno)
+      os.dup2(self.ostream.fileno(), self.orig_stream_fileno)
+
+  def __exit__(self, type, value, traceback):
+    if self.ostream is not None:
+      os.close(self.orig_stream_fileno)
+      os.dup2(self.orig_stream_dup, self.orig_stream_fileno)
+      os.close(self.orig_stream_dup)
+      self.ostream.close()
+%}

source/sentencepiece/sentencepiece_model_pb2.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: sentencepiece_model.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_model_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'H\003'
+  _TRAINERSPEC.fields_by_name['mining_sentence_size']._options = None
+  _TRAINERSPEC.fields_by_name['mining_sentence_size']._serialized_options = b'\030\001'
+  _TRAINERSPEC.fields_by_name['training_sentence_size']._options = None
+  _TRAINERSPEC.fields_by_name['training_sentence_size']._serialized_options = b'\030\001'
+  _TRAINERSPEC._serialized_start=45
+  _TRAINERSPEC._serialized_end=1581
+  _TRAINERSPEC_MODELTYPE._serialized_start=1517
+  _TRAINERSPEC_MODELTYPE._serialized_end=1570
+  _NORMALIZERSPEC._serialized_start=1584
+  _NORMALIZERSPEC._serialized_end=1793
+  _SELFTESTDATA._serialized_start=1795
+  _SELFTESTDATA._serialized_end=1916
+  _SELFTESTDATA_SAMPLE._serialized_start=1864
+  _SELFTESTDATA_SAMPLE._serialized_end=1905
+  _MODELPROTO._serialized_start=1919
+  _MODELPROTO._serialized_end=2429
+  _MODELPROTO_SENTENCEPIECE._serialized_start=2208
+  _MODELPROTO_SENTENCEPIECE._serialized_end=2418
+  _MODELPROTO_SENTENCEPIECE_TYPE._serialized_start=2323
+  _MODELPROTO_SENTENCEPIECE_TYPE._serialized_end=2407
+# @@protoc_insertion_point(module_scope)

source/sentencepiece/sentencepiece_pb2.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: sentencepiece.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13sentencepiece.proto\x12\rsentencepiece\"\xdf\x01\n\x11SentencePieceText\x12\x0c\n\x04text\x18\x01 \x01(\t\x12>\n\x06pieces\x18\x02 \x03(\x0b\x32..sentencepiece.SentencePieceText.SentencePiece\x12\r\n\x05score\x18\x03 \x01(\x02\x1a\x62\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\r\x12\x0f\n\x07surface\x18\x03 \x01(\t\x12\r\n\x05\x62\x65gin\x18\x04 \x01(\r\x12\x0b\n\x03\x65nd\x18\x05 \x01(\r*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"J\n\x16NBestSentencePieceText\x12\x30\n\x06nbests\x18\x01 \x03(\x0b\x32 .sentencepiece.SentencePieceTextB\x02H\x03')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'H\003'
+  _SENTENCEPIECETEXT._serialized_start=39
+  _SENTENCEPIECETEXT._serialized_end=262
+  _SENTENCEPIECETEXT_SENTENCEPIECE._serialized_start=153
+  _SENTENCEPIECETEXT_SENTENCEPIECE._serialized_end=251
+  _NBESTSENTENCEPIECETEXT._serialized_start=264
+  _NBESTSENTENCEPIECETEXT._serialized_end=338
+# @@protoc_insertion_point(module_scope)

source/sentry_sdk-2.53.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

source/sentry_sdk-2.53.0.dist-info/METADATA

@@ -0,0 +1,268 @@
+Metadata-Version: 2.4
+Name: sentry-sdk
+Version: 2.53.0
+Summary: Python client for Sentry (https://sentry.io)
+Home-page: https://github.com/getsentry/sentry-python
+Author: Sentry Team and Contributors
+Author-email: hello@sentry.io
+License: MIT
+Project-URL: Documentation, https://docs.sentry.io/platforms/python/
+Project-URL: Changelog, https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: urllib3>=1.26.11
+Requires-Dist: certifi
+Provides-Extra: aiohttp
+Requires-Dist: aiohttp>=3.5; extra == "aiohttp"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.16; extra == "anthropic"
+Provides-Extra: arq
+Requires-Dist: arq>=0.23; extra == "arq"
+Provides-Extra: asyncpg
+Requires-Dist: asyncpg>=0.23; extra == "asyncpg"
+Provides-Extra: beam
+Requires-Dist: apache-beam>=2.12; extra == "beam"
+Provides-Extra: bottle
+Requires-Dist: bottle>=0.12.13; extra == "bottle"
+Provides-Extra: celery
+Requires-Dist: celery>=3; extra == "celery"
+Provides-Extra: celery-redbeat
+Requires-Dist: celery-redbeat>=2; extra == "celery-redbeat"
+Provides-Extra: chalice
+Requires-Dist: chalice>=1.16.0; extra == "chalice"
+Provides-Extra: clickhouse-driver
+Requires-Dist: clickhouse-driver>=0.2.0; extra == "clickhouse-driver"
+Provides-Extra: django
+Requires-Dist: django>=1.8; extra == "django"
+Provides-Extra: falcon
+Requires-Dist: falcon>=1.4; extra == "falcon"
+Provides-Extra: fastapi
+Requires-Dist: fastapi>=0.79.0; extra == "fastapi"
+Provides-Extra: flask
+Requires-Dist: flask>=0.11; extra == "flask"
+Requires-Dist: blinker>=1.1; extra == "flask"
+Requires-Dist: markupsafe; extra == "flask"
+Provides-Extra: grpcio
+Requires-Dist: grpcio>=1.21.1; extra == "grpcio"
+Requires-Dist: protobuf>=3.8.0; extra == "grpcio"
+Provides-Extra: http2
+Requires-Dist: httpcore[http2]==1.*; extra == "http2"
+Provides-Extra: httpx
+Requires-Dist: httpx>=0.16.0; extra == "httpx"
+Provides-Extra: huey
+Requires-Dist: huey>=2; extra == "huey"
+Provides-Extra: huggingface-hub
+Requires-Dist: huggingface_hub>=0.22; extra == "huggingface-hub"
+Provides-Extra: langchain
+Requires-Dist: langchain>=0.0.210; extra == "langchain"
+Provides-Extra: langgraph
+Requires-Dist: langgraph>=0.6.6; extra == "langgraph"
+Provides-Extra: launchdarkly
+Requires-Dist: launchdarkly-server-sdk>=9.8.0; extra == "launchdarkly"
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.77.5; extra == "litellm"
+Provides-Extra: litestar
+Requires-Dist: litestar>=2.0.0; extra == "litestar"
+Provides-Extra: loguru
+Requires-Dist: loguru>=0.5; extra == "loguru"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.15.0; extra == "mcp"
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == "openai"
+Requires-Dist: tiktoken>=0.3.0; extra == "openai"
+Provides-Extra: openfeature
+Requires-Dist: openfeature-sdk>=0.7.1; extra == "openfeature"
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-distro>=0.35b0; extra == "opentelemetry"
+Provides-Extra: opentelemetry-experimental
+Requires-Dist: opentelemetry-distro; extra == "opentelemetry-experimental"
+Provides-Extra: opentelemetry-otlp
+Requires-Dist: opentelemetry-distro[otlp]>=0.35b0; extra == "opentelemetry-otlp"
+Provides-Extra: pure-eval
+Requires-Dist: pure_eval; extra == "pure-eval"
+Requires-Dist: executing; extra == "pure-eval"
+Requires-Dist: asttokens; extra == "pure-eval"
+Provides-Extra: pydantic-ai
+Requires-Dist: pydantic-ai>=1.0.0; extra == "pydantic-ai"
+Provides-Extra: pymongo
+Requires-Dist: pymongo>=3.1; extra == "pymongo"
+Provides-Extra: pyspark
+Requires-Dist: pyspark>=2.4.4; extra == "pyspark"
+Provides-Extra: quart
+Requires-Dist: quart>=0.16.1; extra == "quart"
+Requires-Dist: blinker>=1.1; extra == "quart"
+Provides-Extra: rq
+Requires-Dist: rq>=0.6; extra == "rq"
+Provides-Extra: sanic
+Requires-Dist: sanic>=0.8; extra == "sanic"
+Provides-Extra: sqlalchemy
+Requires-Dist: sqlalchemy>=1.2; extra == "sqlalchemy"
+Provides-Extra: starlette
+Requires-Dist: starlette>=0.19.1; extra == "starlette"
+Provides-Extra: starlite
+Requires-Dist: starlite>=1.48; extra == "starlite"
+Provides-Extra: statsig
+Requires-Dist: statsig>=0.55.3; extra == "statsig"
+Provides-Extra: tornado
+Requires-Dist: tornado>=6; extra == "tornado"
+Provides-Extra: unleash
+Requires-Dist: UnleashClient>=6.0.1; extra == "unleash"
+Provides-Extra: google-genai
+Requires-Dist: google-genai>=1.29.0; extra == "google-genai"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<a href="https://sentry.io/?utm_source=github&utm_medium=logo" target="_blank">
+  <img src="https://sentry-brand.storage.googleapis.com/github-banners/github-sdk-python.png" alt="Sentry for Python">
+</a>
+<div align="center">
+
+_Bad software is everywhere, and we're tired of it. Sentry is on a mission to help developers write better software faster, so we can get back to enjoying technology. If you want to join us
+[<kbd>**Check out our open positions**</kbd>](https://sentry.io/careers/)_.
+
+[![Discord](https://img.shields.io/discord/621778831602221064?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb)](https://discord.com/invite/Ww9hbqr)
+[![X Follow](https://img.shields.io/twitter/follow/sentry?label=sentry&style=social)](https://x.com/intent/follow?screen_name=sentry)
+[![PyPi page link -- version](https://img.shields.io/pypi/v/sentry-sdk.svg)](https://pypi.python.org/pypi/sentry-sdk)
+<img src="https://img.shields.io/badge/python-3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14-blue.svg" alt="python">
+[![Build Status](https://github.com/getsentry/sentry-python/actions/workflows/ci.yml/badge.svg)](https://github.com/getsentry/sentry-python/actions/workflows/ci.yml)
+
+<br/>
+
+</div>
+
+
+# Official Sentry SDK for Python
+
+Welcome to the official Python SDK for **[Sentry](http://sentry.io/)**.
+
+
+## 📦 Getting Started
+
+### Prerequisites
+
+You need a Sentry [account](https://sentry.io/signup/) and [project](https://docs.sentry.io/product/projects/).
+
+### Installation
+
+Getting Sentry into your project is straightforward. Just run this command in your terminal:
+
+```bash
+pip install --upgrade sentry-sdk
+```
+
+### Basic Configuration
+
+Here's a quick configuration example to get Sentry up and running:
+
+```python
+import sentry_sdk
+
+sentry_sdk.init(
+    "https://12927b5f211046b575ee51fd8b1ac34f@o1.ingest.sentry.io/1",  # Your DSN here
+
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of traces for performance monitoring.
+    traces_sample_rate=1.0,
+)
+```
+
+With this configuration, Sentry will monitor for exceptions and performance issues.
+
+### Quick Usage Example
+
+To generate some events that will show up in Sentry, you can log messages or capture errors:
+
+```python
+import sentry_sdk
+sentry_sdk.init(...)  # same as above
+
+sentry_sdk.capture_message("Hello Sentry!")  # You'll see this in your Sentry dashboard.
+
+raise ValueError("Oops, something went wrong!")  # This will create an error event in Sentry.
+```
+
+
+## 📚 Documentation
+
+For more details on advanced usage, integrations, and customization, check out the full documentation on [https://docs.sentry.io](https://docs.sentry.io/).
+
+
+## 🧩 Integrations
+
+Sentry integrates with a ton of popular Python libraries and frameworks, including [FastAPI](https://docs.sentry.io/platforms/python/integrations/fastapi/), [Django](https://docs.sentry.io/platforms/python/integrations/django/), [Celery](https://docs.sentry.io/platforms/python/integrations/celery/), [OpenAI](https://docs.sentry.io/platforms/python/integrations/openai/) and many, many more.  Check out the [full list of integrations](https://docs.sentry.io/platforms/python/integrations/) to get the full picture.
+
+
+## 🚧 Migrating Between Versions?
+
+### From `1.x` to `2.x`
+
+If you're using the older `1.x` version of the SDK, now's the time to upgrade to `2.x`. It includes significant upgrades and new features. Check our [migration guide](https://docs.sentry.io/platforms/python/migration/1.x-to-2.x) for assistance.
+
+### From `raven-python`
+
+Using the legacy `raven-python` client? It's now in maintenance mode, and we recommend migrating to the new SDK for an improved experience. Get all the details in our [migration guide](https://docs.sentry.io/platforms/python/migration/raven-to-sentry-sdk/).
+
+
+## 🙌 Want to Contribute?
+
+We'd love your help in improving the Sentry SDK! Whether it's fixing bugs, adding features, writing new integrations, or enhancing documentation, every contribution is valuable.
+
+For details on how to contribute, please read our [contribution guide](CONTRIBUTING.md) and explore the [open issues](https://github.com/getsentry/sentry-python/issues).
+
+
+## 🛟 Need Help?
+
+If you encounter issues or need help setting up or configuring the SDK, don't hesitate to reach out to the [Sentry Community on Discord](https://discord.com/invite/Ww9hbqr). There is a ton of great people there ready to help!
+
+
+## 🔗 Resources
+
+Here are all resources to help you make the most of Sentry:
+
+- [Documentation](https://docs.sentry.io/platforms/python/) - Official documentation to get started.
+- [Discord](https://discord.com/invite/Ww9hbqr) - Join our Discord community.
+- [X/Twitter](https://x.com/intent/follow?screen_name=sentry) -  Follow us on X (Twitter) for updates.
+- [Stack Overflow](https://stackoverflow.com/questions/tagged/sentry) - Questions and answers related to Sentry.
+
+<a name="license"></a>
+## 📃 License
+
+The SDK is open-source and available under the MIT license. Check out the [LICENSE](LICENSE) file for more information.
+
+
+## 😘 Contributors
+
+Thanks to everyone who has helped improve the SDK!
+
+<a href="https://github.com/getsentry/sentry-python/graphs/contributors">
+  <img src="https://contributors-img.web.app/image?repo=getsentry/sentry-python" />
+</a>

source/sentry_sdk-2.53.0.dist-info/RECORD

@@ -0,0 +1,386 @@
+sentry_sdk-2.53.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+sentry_sdk-2.53.0.dist-info/METADATA,sha256=rdAnlprh5icT8lL9Qh0lJQw2I6rQOBYbmwG-1XWwXNA,10815
+sentry_sdk-2.53.0.dist-info/RECORD,,
+sentry_sdk-2.53.0.dist-info/WHEEL,sha256=uC7DnXjtoKy23CNUXRq6ohLsm8FbS2C_ECfYYzqZHVY,109
+sentry_sdk-2.53.0.dist-info/entry_points.txt,sha256=qacZEz40UspQZD1IukCXykx0JtImqGDOctS5KfOLTko,91
+sentry_sdk-2.53.0.dist-info/licenses/LICENSE,sha256=KhQNZg9GKBL6KQvHQNBGMxJsXsRdhLebVp4Sew7t3Qs,1093
+sentry_sdk-2.53.0.dist-info/top_level.txt,sha256=XrQz30XE9FKXSY_yGLrd9bsv2Rk390GTDJOSujYaMxI,11
+sentry_sdk/__init__.py,sha256=cnZoQ9y329brs-cdzIVtxbO1-o9AIrKk8VTVyZNJs1A,1410
+sentry_sdk/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/__pycache__/_batcher.cpython-312.pyc,,
+sentry_sdk/__pycache__/_compat.cpython-312.pyc,,
+sentry_sdk/__pycache__/_init_implementation.cpython-312.pyc,,
+sentry_sdk/__pycache__/_log_batcher.cpython-312.pyc,,
+sentry_sdk/__pycache__/_lru_cache.cpython-312.pyc,,
+sentry_sdk/__pycache__/_metrics_batcher.cpython-312.pyc,,
+sentry_sdk/__pycache__/_queue.cpython-312.pyc,,
+sentry_sdk/__pycache__/_span_batcher.cpython-312.pyc,,
+sentry_sdk/__pycache__/_types.cpython-312.pyc,,
+sentry_sdk/__pycache__/_werkzeug.cpython-312.pyc,,
+sentry_sdk/__pycache__/api.cpython-312.pyc,,
+sentry_sdk/__pycache__/attachments.cpython-312.pyc,,
+sentry_sdk/__pycache__/client.cpython-312.pyc,,
+sentry_sdk/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/__pycache__/debug.cpython-312.pyc,,
+sentry_sdk/__pycache__/envelope.cpython-312.pyc,,
+sentry_sdk/__pycache__/feature_flags.cpython-312.pyc,,
+sentry_sdk/__pycache__/hub.cpython-312.pyc,,
+sentry_sdk/__pycache__/logger.cpython-312.pyc,,
+sentry_sdk/__pycache__/metrics.cpython-312.pyc,,
+sentry_sdk/__pycache__/monitor.cpython-312.pyc,,
+sentry_sdk/__pycache__/scope.cpython-312.pyc,,
+sentry_sdk/__pycache__/scrubber.cpython-312.pyc,,
+sentry_sdk/__pycache__/serializer.cpython-312.pyc,,
+sentry_sdk/__pycache__/session.cpython-312.pyc,,
+sentry_sdk/__pycache__/sessions.cpython-312.pyc,,
+sentry_sdk/__pycache__/spotlight.cpython-312.pyc,,
+sentry_sdk/__pycache__/traces.cpython-312.pyc,,
+sentry_sdk/__pycache__/tracing.cpython-312.pyc,,
+sentry_sdk/__pycache__/tracing_utils.cpython-312.pyc,,
+sentry_sdk/__pycache__/transport.cpython-312.pyc,,
+sentry_sdk/__pycache__/types.cpython-312.pyc,,
+sentry_sdk/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/__pycache__/worker.cpython-312.pyc,,
+sentry_sdk/_batcher.py,sha256=3Elbey1gpWzYSo-Dr5xxP00QpYcRYb8FGYqei9U_mdE,4004
+sentry_sdk/_compat.py,sha256=tSI9LTAlSw7x_putvg7qg5SSdo7W1EwpbYiB_godUMQ,3065
+sentry_sdk/_init_implementation.py,sha256=xualjTVpMB8XbJUX-30eAnuiyPfDeoeZoer3iIl0mH4,2491
+sentry_sdk/_log_batcher.py,sha256=6Y3VNP6a65t1gzND9Rz9356xj2ZXOI1rB8Wq2De7Cpc,1827
+sentry_sdk/_lru_cache.py,sha256=YSnCmL1qArB6_dsfcwKZgO4eJhBak87HqjqbUzBNQ7A,1167
+sentry_sdk/_metrics_batcher.py,sha256=c7aw1sG7rldcS4XrpKAX59AGMXZ74Vx9Ja7peQo7MPc,1273
+sentry_sdk/_queue.py,sha256=bluPs51jz8d3w74aSolOZYc3jnwmyTGPKvKxv7Z2LL0,11244
+sentry_sdk/_span_batcher.py,sha256=EscEhZvzyJIby2taeOfRL7m6hm2RwptdmXwk0b8k36o,4406
+sentry_sdk/_types.py,sha256=bcDjVByhfqJoYB2deda3oZBYv73gRanJ3WfFM2WagXM,11018
+sentry_sdk/_werkzeug.py,sha256=vQL3Z_q3OBQ8vEby2ktiY7Ey8szMDWZsUe-BGJ9A3oY,3716
+sentry_sdk/ai/__init__.py,sha256=L2EAYEx9075dUVZGEB_CBxSgwaLeYLFWhBYfIEeBDRg,218
+sentry_sdk/ai/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/ai/__pycache__/_openai_completions_api.cpython-312.pyc,,
+sentry_sdk/ai/__pycache__/_openai_responses_api.cpython-312.pyc,,
+sentry_sdk/ai/__pycache__/monitoring.cpython-312.pyc,,
+sentry_sdk/ai/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/ai/_openai_completions_api.py,sha256=MeLmyCCd-PQLWRN4RHgCm7bK6QXBjU74Df0ZGHDDDCg,1821
+sentry_sdk/ai/_openai_responses_api.py,sha256=xN2fbP2hNibB-kW0mbA7L9s_g_JrYhyra7V4S9xF8kU,656
+sentry_sdk/ai/monitoring.py,sha256=PRYQXdKh2CdIttIKcCWhQKebGHlBHn6b0lKWAmhK7MQ,5539
+sentry_sdk/ai/utils.py,sha256=VtyghTEHI_3KRrEltYpLlIOO5FX5eD_Og1l7sqA9mx4,23837
+sentry_sdk/api.py,sha256=p4ABg2gX0Lx5T6nbi-D8LIo-p3lB37IcI91I_bEoJYg,14668
+sentry_sdk/attachments.py,sha256=fvmTzYvG2a2mjNzTzcja28ize87kEEZMtXwK1rmXpBc,3023
+sentry_sdk/client.py,sha256=jhdrYnioKRqwZy66VgqrwGNyTewajkd1stPU0iZqZLA,38867
+sentry_sdk/consts.py,sha256=ndv79s9CYhXd5N1S5a_F6MiO2lAimC2k69aSZYQU2kM,55042
+sentry_sdk/crons/__init__.py,sha256=3Zt6g1-pZZ12uRKKsC8QLm3XgJ4K1VYxgVpNNUygOZY,221
+sentry_sdk/crons/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/crons/__pycache__/api.cpython-312.pyc,,
+sentry_sdk/crons/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/crons/__pycache__/decorator.cpython-312.pyc,,
+sentry_sdk/crons/api.py,sha256=gwtNjHZxZy51piEwYrnrYg3bZkifrdkSRlDAZYC30Kg,1679
+sentry_sdk/crons/consts.py,sha256=dXqJk5meBSu5rjlGpqAOlkpACnuUi7svQnAFoy1ZNUU,87
+sentry_sdk/crons/decorator.py,sha256=4fYEHYlhdtKyhzJr1HZZ7g2ierEJ3ULZe4N7a3jqbYE,3834
+sentry_sdk/debug.py,sha256=63f8uUowpbZ75E8q-r_8Tdymsv9jXZn1VMh287VcWhk,959
+sentry_sdk/envelope.py,sha256=q0CtNI9IgYPTlviQyp8HB8aeIxLVcklMIqaQ-LU_HpU,9652
+sentry_sdk/feature_flags.py,sha256=J9G-nnQiXimYbhkPNDGz_CjAccYlbXS9KgybExBZjUI,2136
+sentry_sdk/hub.py,sha256=sVkOrpjU0cNhxOFtWfxA8Nc_3T8hiGH-mVFZa4tX1Us,25147
+sentry_sdk/integrations/__init__.py,sha256=U52rckcGzXoe73On-T70R64feEIg1ir1O1NSSkQvHzg,12674
+sentry_sdk/integrations/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/_asgi_common.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/_wsgi_common.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/aiohttp.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/anthropic.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/argv.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/ariadne.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/arq.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/asgi.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/asyncio.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/asyncpg.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/atexit.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/aws_lambda.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/beam.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/boto3.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/bottle.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/chalice.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/clickhouse_driver.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/cloud_resource_context.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/cohere.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/dedupe.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/dramatiq.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/excepthook.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/executing.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/falcon.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/fastapi.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/flask.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/gcp.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/gnu_backtrace.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/gql.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/graphene.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/httpx.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/huey.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/huggingface_hub.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/langchain.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/langgraph.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/launchdarkly.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/litellm.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/litestar.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/logging.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/loguru.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/mcp.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/modules.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/openai.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/openfeature.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/otlp.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/pure_eval.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/pymongo.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/pyramid.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/quart.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/ray.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/rq.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/rust_tracing.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/sanic.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/serverless.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/socket.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/sqlalchemy.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/starlette.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/starlite.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/statsig.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/stdlib.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/strawberry.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/sys_exit.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/threading.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/tornado.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/trytond.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/typer.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/unleash.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/unraisablehook.cpython-312.pyc,,
+sentry_sdk/integrations/__pycache__/wsgi.cpython-312.pyc,,
+sentry_sdk/integrations/_asgi_common.py,sha256=qY2nH21YGtQ1EufzdrovHtGZ-ZaMbSzYNKCo8z-kD44,3145
+sentry_sdk/integrations/_wsgi_common.py,sha256=ffyvbYSYtVNo8wkzx7tt1T7anuMiu2ixApZmC0O9NH4,7282
+sentry_sdk/integrations/aiohttp.py,sha256=vQrqPm_IVgvgQo4PLEyDNrvA5oloz1xALOFtVDTtOpE,12930
+sentry_sdk/integrations/anthropic.py,sha256=CDLZf73v8HBj-OxDzJHlgmecXFHeIIYKoem2kdHmKP4,21222
+sentry_sdk/integrations/argv.py,sha256=3dyTYcNVhx-EoCxzZI8-JO1mxydBo_zHxIKHpLJgJcc,877
+sentry_sdk/integrations/ariadne.py,sha256=u1X6QEcXfwBXa4mmHkgcj8XMoowwBGIWQQYAsItpjwg,5792
+sentry_sdk/integrations/arq.py,sha256=3hs8Y41X3rjxCd769CHjUBYQe3YgULtbX6WQQB5-kwQ,7906
+sentry_sdk/integrations/asgi.py,sha256=AoJcWDRkmm5vqSMH44V2zx00BiT9Kzqfo3pd3Eyi6D8,12687
+sentry_sdk/integrations/asyncio.py,sha256=nszg5HkxWLoTzfS3EZ9hBzsCC7JM8JKOBFXbNXnCDpI,6994
+sentry_sdk/integrations/asyncpg.py,sha256=T0Ijpln-xoQCuY0AS15dYZK2_1XCdKYT1J9EfdSQURo,6578
+sentry_sdk/integrations/atexit.py,sha256=eUcgC6f7WoqJadxDbn76dBv6zoS97kgyxshjGBNK1us,1567
+sentry_sdk/integrations/aws_lambda.py,sha256=HDvJk2EpOjS0_2wkTXc9lQr2w-huT7v3j4_-Pxq0KPw,17895
+sentry_sdk/integrations/beam.py,sha256=j-BzEzAHXk12HJ8izUyIOaM02d29BZUpYldBMCq8vdI,5092
+sentry_sdk/integrations/boto3.py,sha256=4HLGehOoU8rc-85yL5ATUUZL9x_TGYjPkNYHOcT4Z2w,4341
+sentry_sdk/integrations/bottle.py,sha256=Im8xqoQvrygqKHsK-rtaX41gOaY_hxb4anjgJnZQGu4,6404
+sentry_sdk/integrations/celery/__init__.py,sha256=LTWUX6mEsf93fITkcLUV1exm7Rk12DY6JUGKH_gPXCI,18497
+sentry_sdk/integrations/celery/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/celery/__pycache__/beat.cpython-312.pyc,,
+sentry_sdk/integrations/celery/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/celery/beat.py,sha256=RCG2yAyJSyWtzDGw6B1M_w3TXDvPnznAt1dhfFcLBDI,8839
+sentry_sdk/integrations/celery/utils.py,sha256=KyaM868RGak0JeFHkQVYtR1CkKRNpSD4_bbpisF5jqo,1152
+sentry_sdk/integrations/chalice.py,sha256=nd_J4MnPZzoVdH4M5HV_-ljeS8fbxe_5DXmKJm6zqSI,4663
+sentry_sdk/integrations/clickhouse_driver.py,sha256=mNYW99LlIOCk45aUeELY9cuHTfBorhwV0rGn2c_25EY,5905
+sentry_sdk/integrations/cloud_resource_context.py,sha256=J_3Zd9czyI2NuWy09BEwGKzw80rQlVlJ1CQqTvJ5m18,7638
+sentry_sdk/integrations/cohere.py,sha256=QxgvN7fy385-EwUPYFup98UiMxxg_zrE8h0yCfI5iT8,9495
+sentry_sdk/integrations/dedupe.py,sha256=EApcjbAp_uEmsOSb0m64JNGaKUEqB46YCYX8AgTDO24,1903
+sentry_sdk/integrations/django/__init__.py,sha256=T2N0ZnN4aI4AGJvoyNtYBBXtAKPrN3suuQPQi1Ks_r8,26421
+sentry_sdk/integrations/django/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/asgi.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/caching.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/middleware.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/signals_handlers.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/tasks.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/templates.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/transactions.cpython-312.pyc,,
+sentry_sdk/integrations/django/__pycache__/views.cpython-312.pyc,,
+sentry_sdk/integrations/django/asgi.py,sha256=xi4ceuIgjFsvyBwzGOwHj_siLHqeYHuL5oLfPqDxPKQ,8436
+sentry_sdk/integrations/django/caching.py,sha256=gXRDSZigV1SETmxDUe8GA53Tha2fo0ZAFOofuorQD30,6980
+sentry_sdk/integrations/django/middleware.py,sha256=kSN4ORVUuO8Yvt0sC6YS8br2gk6gLmBwysIFsN9z2js,5917
+sentry_sdk/integrations/django/signals_handlers.py,sha256=E1r1sYzKQHko6fD8bPC7_rXGH5kBVDwmE2sutWuGAwQ,3062
+sentry_sdk/integrations/django/tasks.py,sha256=Bu_5jxvFNcAk9N8PedxhWP5Fq-Mj8zVRYdDRsd-6eJA,1151
+sentry_sdk/integrations/django/templates.py,sha256=NZuSVDU0srkF_La5BpEIGqonCbK-qLKMMI3bqj2za3w,5695
+sentry_sdk/integrations/django/transactions.py,sha256=zjsJgVFJZX-c5H_I-TXeLrvrw3OvgAWgmxYO5nTdhfQ,4918
+sentry_sdk/integrations/django/views.py,sha256=Cz2H_yeZkPixOksVoi5urp6Xeq5p0YdDsBWz5aEJan4,3256
+sentry_sdk/integrations/dramatiq.py,sha256=OIq0lIUbabt9widLKGkyjhvx9MKXNF9atP1gTsLheuQ,7438
+sentry_sdk/integrations/excepthook.py,sha256=lhbIwr1eIj3k8shtmUGDOWI7Qr-OaT5kbgeHj-F217g,2371
+sentry_sdk/integrations/executing.py,sha256=O5R4pw999PIisy7uksrMkRx0SsmfBFKI1BEpI3cTDz4,1981
+sentry_sdk/integrations/falcon.py,sha256=zWKFTzoOY92BryGKsjKDAhVxFDhyop4G02js6OlmIXY,9277
+sentry_sdk/integrations/fastapi.py,sha256=eebHCt2FiR7YQCaRJVOnXurltMZ4phRCZaL6BXSEpVI,4485
+sentry_sdk/integrations/flask.py,sha256=KGG2Jbcw9SdwADJLg5GmBnVq3VywK7qQwdPv97RldmA,8519
+sentry_sdk/integrations/gcp.py,sha256=Fao0nrrBpiG8_GHygzh6VvmM8bpJq0mkvgTwu5LT1a4,8400
+sentry_sdk/integrations/gnu_backtrace.py,sha256=XND4pkVeprhDdvPgOi8pBgQQ9g3CtPA-wJK0e0ex1XE,2771
+sentry_sdk/integrations/google_genai/__init__.py,sha256=Z-NX8wKGkSwlQupCcCgsarGV3qtpmeiT_wGNqpxweSM,14093
+sentry_sdk/integrations/google_genai/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/google_genai/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/google_genai/__pycache__/streaming.cpython-312.pyc,,
+sentry_sdk/integrations/google_genai/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/google_genai/consts.py,sha256=nqHKKSyGixrSoozA06BGVBFaUCsvZlvGoubUZGI1kB8,559
+sentry_sdk/integrations/google_genai/streaming.py,sha256=JqhTcAX3DtEsZ033AXsnA0KAzQe_jqca-7cztTKDcRI,5416
+sentry_sdk/integrations/google_genai/utils.py,sha256=WCTyevkQF2CdGpdlidONJSWSdDmnqEa0dcaiRX2luuk,34850
+sentry_sdk/integrations/gql.py,sha256=oLssOmHYQ4e0OQ1ITer8TynvyLXtDUoM9X29-IsVAkI,5003
+sentry_sdk/integrations/graphene.py,sha256=11XRvAsSgvCLcfF1xvdSeAf0vvJm72hfcE3toAS8jZI,5066
+sentry_sdk/integrations/grpc/__init__.py,sha256=0Y3neOWwES1PsWJsluPF_2_VatFlVvg7IUjTYYdCs-U,6253
+sentry_sdk/integrations/grpc/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/__pycache__/client.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/__pycache__/server.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/aio/__init__.py,sha256=2rgrliowpPfLLw40_2YU6ixSzIu_3f8NN3TRplzc8S8,141
+sentry_sdk/integrations/grpc/aio/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/aio/__pycache__/client.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/aio/__pycache__/server.cpython-312.pyc,,
+sentry_sdk/integrations/grpc/aio/client.py,sha256=2GNrTWJt4iZ9dK-M3_0-sVWlUcfqKItHZI5u-uzesUM,3507
+sentry_sdk/integrations/grpc/aio/server.py,sha256=cRBvz9GOJemvNOsCB9ymTUX70nhjljfzvX0bfR8Wc_o,3954
+sentry_sdk/integrations/grpc/client.py,sha256=s7yLcbYgnq1-bdh6jvSOv6NwKTx0m9tLPt0q4qKI7kw,3404
+sentry_sdk/integrations/grpc/consts.py,sha256=NpsN5gKWDmtGtVK_L5HscgFZBHqjOpmLJLGKyh8GZBA,31
+sentry_sdk/integrations/grpc/server.py,sha256=h6vEibthsZB6hWNmqT50UlBaddEZSdbXio_rYhighW0,2470
+sentry_sdk/integrations/httpx.py,sha256=MxoT_D8n902awI2hq-lHpxXkOvNwGZFuG9ap4l4VxL4,5315
+sentry_sdk/integrations/huey.py,sha256=GOslGxuYfrdPLqx_4MWOH_tdwwnbD72SkTdc-P8uF80,5348
+sentry_sdk/integrations/huggingface_hub.py,sha256=axCqdDlmYn7wANHgVBo3MDv9QgBr0MWj3lBYhPJJX7E,14966
+sentry_sdk/integrations/langchain.py,sha256=Oa-YJ2WoHX8Huyrzs4aQ2xlaupFR5Ft8FC4_0b_zvBs,42903
+sentry_sdk/integrations/langgraph.py,sha256=xb9InjYjd4jAQECWddl30QGRoHRbbhoYBeHscsM7C1A,13301
+sentry_sdk/integrations/launchdarkly.py,sha256=RSTCAUOJCo6Npr4V3-7-vSOaZ531Jj1LNU5IftXL6_w,1918
+sentry_sdk/integrations/litellm.py,sha256=1CLM3CBdIv0YZSloDPCfemIdlufNG41eWvCBRG_AFow,11550
+sentry_sdk/integrations/litestar.py,sha256=oJIz1vv2JgIT_AriKXDTQvQjkMOmwKfT68DFrCfdJ-8,11698
+sentry_sdk/integrations/logging.py,sha256=eLjza7xDrfBszOH4gDHxPtSM3WfiWsw9WGR3GUfnA-A,13739
+sentry_sdk/integrations/loguru.py,sha256=2UaLLzXVEOV0cYxOU5n9GpkmNtC_bA1KzCzEnrqIilE,6505
+sentry_sdk/integrations/mcp.py,sha256=4t7cVGwqjoqyNSd4HrwzjfM7fw2B5HYWRJoT6wOw6AA,22192
+sentry_sdk/integrations/modules.py,sha256=7kmS9T-werRkKN0w51skyDmTMuZbdz9L28RgXPsoDeU,786
+sentry_sdk/integrations/openai.py,sha256=8UquHKCu17tS6Afs90HLDa4mpx-7ErRl4nUh1OKwdbE,33167
+sentry_sdk/integrations/openai_agents/__init__.py,sha256=0FHafAmsKZKNtijv3ZRxS9b2xoUvfHbbgFuWZ3eSbRo,10709
+sentry_sdk/integrations/openai_agents/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/consts.py,sha256=PTb3vlqkuMPktu21ALK72o5WMIX4-cewTEiTRdHKFdQ,38
+sentry_sdk/integrations/openai_agents/patches/__init__.py,sha256=62qvLBPGajoxGi4C41eBOztIiB8MGRpmP2i3GSD_22k,383
+sentry_sdk/integrations/openai_agents/patches/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/__pycache__/agent_run.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/__pycache__/error_tracing.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/__pycache__/models.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/__pycache__/runner.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/__pycache__/tools.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/patches/agent_run.py,sha256=OhLQLEB7LQEMRs6BkQ0jfyY6QaZ-lhP5yZ_b44KTRV0,8257
+sentry_sdk/integrations/openai_agents/patches/error_tracing.py,sha256=Fzs4oAyGWnmbaeLMMRnr9m6F6M2N9Cm0i4GSPsJbNqI,2237
+sentry_sdk/integrations/openai_agents/patches/models.py,sha256=mKUbs01jhNIs06fKgP6nraRB-5m_UvCGkvK0Xu4cuDg,7278
+sentry_sdk/integrations/openai_agents/patches/runner.py,sha256=gJLrC334MrfcSCxG93__iOh3jcROGWw4YKHXcHJ5dYY,6837
+sentry_sdk/integrations/openai_agents/patches/tools.py,sha256=9I7GOIY-X3WyegTMMq-KIbs5DQ3Qg5JHHtIrEU3eivA,2823
+sentry_sdk/integrations/openai_agents/spans/__init__.py,sha256=uzqInMjqfA_MtdTJoRRH10Z4cVtVESbi7qc5sC_zr8U,393
+sentry_sdk/integrations/openai_agents/spans/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/__pycache__/agent_workflow.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/__pycache__/ai_client.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/__pycache__/execute_tool.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/__pycache__/handoff.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/__pycache__/invoke_agent.cpython-312.pyc,,
+sentry_sdk/integrations/openai_agents/spans/agent_workflow.py,sha256=lqr87hbaV1GCXX40cGQLSlf3U6V-nmD2zj5kr2w9kmk,459
+sentry_sdk/integrations/openai_agents/spans/ai_client.py,sha256=RbJNDn9DXM4NJLHJBIodRp1GK-kQmclYCD5DgOV2WTI,2265
+sentry_sdk/integrations/openai_agents/spans/execute_tool.py,sha256=ePm9PMnEzoYZW8M04dnDRINV7dRnDKtHa4YFn1InqZY,1612
+sentry_sdk/integrations/openai_agents/spans/handoff.py,sha256=icCbs6HOFrVXY4V1LILCoyV-qxNntIBhgx8AFHW6MGo,723
+sentry_sdk/integrations/openai_agents/spans/invoke_agent.py,sha256=VWB1HmR3oyoTqi3ZrHIHY37ageQnPsap5C81bLcxrOA,3592
+sentry_sdk/integrations/openai_agents/utils.py,sha256=MvbPG-dh5UJzE50y-Q9xUeSuuu_Ot642SvD1H0hIcuY,8910
+sentry_sdk/integrations/openfeature.py,sha256=w6bYjrHJdjIpziEvEBocCbxdryO2eDZGGOoc7g8jNEU,1102
+sentry_sdk/integrations/opentelemetry/__init__.py,sha256=emNL5aAq_NhK0PZmfX_g4GIdvBS6nHqGrjrIgrdC5m8,229
+sentry_sdk/integrations/opentelemetry/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/opentelemetry/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/opentelemetry/__pycache__/integration.cpython-312.pyc,,
+sentry_sdk/integrations/opentelemetry/__pycache__/propagator.cpython-312.pyc,,
+sentry_sdk/integrations/opentelemetry/__pycache__/span_processor.cpython-312.pyc,,
+sentry_sdk/integrations/opentelemetry/consts.py,sha256=6yhH65VVzzIjpU8MRYItLKkskkBsLlbsO-_2MULlEHQ,275
+sentry_sdk/integrations/opentelemetry/integration.py,sha256=VPQlj-tDBgjSRXOxtrxeFpv9Mc_OqX7cC997gMy4CkY,1742
+sentry_sdk/integrations/opentelemetry/propagator.py,sha256=ofVK4IJ9zhSN0cRmw_Cr15IHK0XckmBy8WqA_oaGa5Q,3973
+sentry_sdk/integrations/opentelemetry/span_processor.py,sha256=YK-k88VMoJNrOEXMqYaVM_Y-FywN2hQjdrodLeqGegw,13222
+sentry_sdk/integrations/otlp.py,sha256=5h0hFY19myYSg_buSAutomC_8_BfRtDyHyLE9XD_A8A,7713
+sentry_sdk/integrations/pure_eval.py,sha256=SLN-omknLQWyeH1_3v3Qi-puFCAliYKQqi0wdDt3Keg,4560
+sentry_sdk/integrations/pydantic_ai/__init__.py,sha256=YrUxvNgPIznxzVaniyLTorkkTmlYyosAbA37tFOTR68,1465
+sentry_sdk/integrations/pydantic_ai/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/consts.py,sha256=fxOQ5n_Do8EqqqxtOJm5zyvhQmOV75HACNrt_-zGngs,36
+sentry_sdk/integrations/pydantic_ai/patches/__init__.py,sha256=_RHvjc3436KSwPjzrAdnyascgggxg5e0MQpdHhmiS-U,229
+sentry_sdk/integrations/pydantic_ai/patches/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/patches/__pycache__/agent_run.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/patches/__pycache__/graph_nodes.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/patches/__pycache__/model_request.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/patches/__pycache__/tools.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/patches/agent_run.py,sha256=iGafWMXefgs_1HQfV_rK4Pe5wJex4vLbAdX-O83fxQQ,7345
+sentry_sdk/integrations/pydantic_ai/patches/graph_nodes.py,sha256=TDkIjRUxGGt0UbC3FzgaCJNJN1s682IXoVFO07biL6E,3793
+sentry_sdk/integrations/pydantic_ai/patches/model_request.py,sha256=15uGRv0UXjZcEc9b6Jzx7o9yhToUQfd4KRLZm_OWMm8,1240
+sentry_sdk/integrations/pydantic_ai/patches/tools.py,sha256=soSA-ugtq-EdkkbfXV_mAPC-AfEaCM_6E_c1kvg_IQk,3822
+sentry_sdk/integrations/pydantic_ai/spans/__init__.py,sha256=dTUjvkw7VMOAiSasuAq37q_njvANsUlgfZxgXRKJDDo,243
+sentry_sdk/integrations/pydantic_ai/spans/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/spans/__pycache__/ai_client.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/spans/__pycache__/execute_tool.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/spans/__pycache__/invoke_agent.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/spans/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/pydantic_ai/spans/ai_client.py,sha256=aAEjW2eh317ZFQJwJi2_jJHnWaKtdWdJFkuaU-BFbj8,10508
+sentry_sdk/integrations/pydantic_ai/spans/execute_tool.py,sha256=0k-eIN1tltSOQf2kFdUMGKbyWhKYTNZE8dJF9vkOfnE,1542
+sentry_sdk/integrations/pydantic_ai/spans/invoke_agent.py,sha256=Mbq_7NkqD5oyjY-_AOhblliYNT_Ij6z-MEMfZl4xyyY,5742
+sentry_sdk/integrations/pydantic_ai/spans/utils.py,sha256=VSWLcgjrUwNSR_31TFM5BRG0EsGrc3X8BWIxM4v0HZI,1763
+sentry_sdk/integrations/pydantic_ai/utils.py,sha256=OWFa15_jQ7H4gHfMAUQHmSVclLVrF_DBb7heaPZ89HY,7091
+sentry_sdk/integrations/pymongo.py,sha256=whIHxnsiz245UNT7yKMP1OqfAsBdC7ITy7kuice0uwE,6275
+sentry_sdk/integrations/pyramid.py,sha256=EzK6W2VB4DdDfV_VXnp5ERibM27mVK-JGib3k9Khc4E,7208
+sentry_sdk/integrations/quart.py,sha256=4GTx6E7auhLeYXyY8WoloqSghbeU14Jcz-gfDL4o3aE,7258
+sentry_sdk/integrations/ray.py,sha256=8zLUo2ZFl9KEt88X-yJYVVoDphJUTWRZRT0I5OQ1PCk,5884
+sentry_sdk/integrations/redis/__init__.py,sha256=TzeEWrYcDO9u3q5WXT8J9Zk5Ft9nrWW1PsE3i0ULGUg,1662
+sentry_sdk/integrations/redis/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/_async_common.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/_sync_common.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/consts.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/rb.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/redis.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/redis_cluster.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/redis_py_cluster_legacy.cpython-312.pyc,,
+sentry_sdk/integrations/redis/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/integrations/redis/_async_common.py,sha256=V05_CgSyWBKBLQ4Udkg4LT5ovPAnP5iqsvfeqsmLEwU,4058
+sentry_sdk/integrations/redis/_sync_common.py,sha256=3seE5lTc2fDNLXutpe41Qv-3pdRVbLHFGCQaT9OXzvc,3776
+sentry_sdk/integrations/redis/consts.py,sha256=y2f-FJ7TIkzto01tyjXvbKVSVELVkjZaxj3FG5DZ0hA,480
+sentry_sdk/integrations/redis/modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sentry_sdk/integrations/redis/modules/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/redis/modules/__pycache__/caches.cpython-312.pyc,,
+sentry_sdk/integrations/redis/modules/__pycache__/queries.cpython-312.pyc,,
+sentry_sdk/integrations/redis/modules/caches.py,sha256=cV64P6UWHXolvTtog5jQFHtR4XFd5IqA5vVYXc8xO2E,4020
+sentry_sdk/integrations/redis/modules/queries.py,sha256=lRwn72AUvk5a29n43UEpx6l9ix_xC2htHcl4cHKsdcw,1941
+sentry_sdk/integrations/redis/rb.py,sha256=we3oh4fEhiirglutAllZe55sAoz6vKtaQt1tvLPGoX4,791
+sentry_sdk/integrations/redis/redis.py,sha256=NaaefF_0poC8-9citN6etyAC6WOabxBVOdH_J8FJ7ec,1684
+sentry_sdk/integrations/redis/redis_cluster.py,sha256=rBbj1WRa_-NAR6tTQHKQMF40WjvNb6o6FoojImIE03Y,3530
+sentry_sdk/integrations/redis/redis_py_cluster_legacy.py,sha256=YREwCLBkx_SDVkzNLGNcgCr4yi_mdomlwgKlVU8bgSU,1570
+sentry_sdk/integrations/redis/utils.py,sha256=i818myPF61yl4EIS-t3ptyYCa2IrsaAWeFR0cqS0QNY,3954
+sentry_sdk/integrations/rq.py,sha256=wyerb5iiAJbekL5-kdOpwQzQiJiWIRlX49il1aEnUWg,5278
+sentry_sdk/integrations/rust_tracing.py,sha256=p3aFy8Gs8Dgvvlv9PSrbl7JDYmA5zus-fTichRzTiLM,9101
+sentry_sdk/integrations/sanic.py,sha256=OJ570xdV4luxJY9_b4gutVf8gQhNpqyZHxyT3evydLM,12700
+sentry_sdk/integrations/serverless.py,sha256=eZSd0NPYBm3ZGdYFmDYlXDtAD_Pe_aUGV77bCzJhoEI,1621
+sentry_sdk/integrations/socket.py,sha256=fj-4yzw7uecKDa4Jl_8AN_Nb9KSIvCh3iR8oqaLsYaQ,3164
+sentry_sdk/integrations/spark/__init__.py,sha256=oOewMErnZk2rzNvIlZO6URxQexu9bUJuSLM2m_zECy8,208
+sentry_sdk/integrations/spark/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/integrations/spark/__pycache__/spark_driver.cpython-312.pyc,,
+sentry_sdk/integrations/spark/__pycache__/spark_worker.cpython-312.pyc,,
+sentry_sdk/integrations/spark/spark_driver.py,sha256=mYX1ohjUD7wrc7F16OhvNyyOGqgQ7QM6bFzfBb_MUZY,8896
+sentry_sdk/integrations/spark/spark_worker.py,sha256=PtRzOBjrG0__Au-Uidd8cO5zpx-EJY_D13H9CG900RE,3632
+sentry_sdk/integrations/sqlalchemy.py,sha256=Q6JQxDcwo2U-3ggB_wUhQZps1gzzXz6-Yf20dcCMUWU,4347
+sentry_sdk/integrations/starlette.py,sha256=1cSqN0IARPGi6kLitW93ASlTJIAFFUxsyWnS8QMMzJY,26130
+sentry_sdk/integrations/starlite.py,sha256=GC_FaL-cyjeUtsRIJ29jMR4HlYXRwI-JRYy7GFn5QAI,10451
+sentry_sdk/integrations/statsig.py,sha256=ceSjGymN4Fqdnk8Tz-L5teHz0eNOJtptB_W0Y1VZu2o,1214
+sentry_sdk/integrations/stdlib.py,sha256=2yN0_v-fokZwgfnsKBvcIdl4tQ-QpLUnoQzYyOX_gH4,9376
+sentry_sdk/integrations/strawberry.py,sha256=cTY1L8ihyR5mCiF2H5ZEM59t0BMwNmWqwNkTIuIrFUA,14120
+sentry_sdk/integrations/sys_exit.py,sha256=nLbBnbyPjUgyhsI1VkmTxVAOgdlenf-dFtIqzFTZNOI,2408
+sentry_sdk/integrations/threading.py,sha256=jUsmnGEPoi6XGhuLZP3pWKJCDksul-e7uLwAyPK_vUI,7107
+sentry_sdk/integrations/tornado.py,sha256=jh_tm6UR-m-OPFlTKD4eOYEsyex2Lm9rw7_UpZ-1dCQ,7212
+sentry_sdk/integrations/trytond.py,sha256=FjAKIKDTRmEc1JUFsHkcZhslCOD7RsT-pKYoSg1Qyuk,1750
+sentry_sdk/integrations/typer.py,sha256=hvLL7llTverbhjdC8eU5r0ps39XVf2qI27C35VWyXrE,1833
+sentry_sdk/integrations/unleash.py,sha256=Gz3qAKB0fTtOgzCOJV24ocW1NbdcDkkqijrWDE6Oi7E,1045
+sentry_sdk/integrations/unraisablehook.py,sha256=zThW8f6uFGFfwpPHFXq_pLbnaQRT-1tmltyOPVfTPzM,1717
+sentry_sdk/integrations/wsgi.py,sha256=w6BKB-aCaGKl6tfWlKL2SlifY4ZErQaEH5oCXg_bcFg,10503
+sentry_sdk/logger.py,sha256=3hfleFTl9n6ZlZeb-F8jVVMGoae_SU8fOZd0N6HWbuA,2683
+sentry_sdk/metrics.py,sha256=z8futlRltjsPeTkU_We4HAanxeopDaWtfJTkAumwL68,1470
+sentry_sdk/monitor.py,sha256=YRmomlJySjuuCHLJPgGMhRbHXun651He5QZXUvKWZp0,3443
+sentry_sdk/profiler/__init__.py,sha256=3PI3bHk9RSkkOXZKN84DDedk_7M65EiqqaIGo-DYs0E,1291
+sentry_sdk/profiler/__pycache__/__init__.cpython-312.pyc,,
+sentry_sdk/profiler/__pycache__/continuous_profiler.cpython-312.pyc,,
+sentry_sdk/profiler/__pycache__/transaction_profiler.cpython-312.pyc,,
+sentry_sdk/profiler/__pycache__/utils.cpython-312.pyc,,
+sentry_sdk/profiler/continuous_profiler.py,sha256=lRZa2yF81ByIh1yQVHdsK-Ur5jRMR6aKJuPCkuHn0pI,22550
+sentry_sdk/profiler/transaction_profiler.py,sha256=AJicxLx2aPcnTBm5E6GGx_34j-y2yD907_OsHtN6xyQ,27248
+sentry_sdk/profiler/utils.py,sha256=Y-GN2SoAKa12nxwUQRvI5LTYEWfIkhOvoCPG6B4K_N0,6397
+sentry_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sentry_sdk/scope.py,sha256=HtA_ivY_Syuw2sxXiXCgp1PsRg46CetAuVeth_slfzk,68615
+sentry_sdk/scrubber.py,sha256=pEJECHaWzg0vf93S_uYe9PgStTImDXjLFu0pzIJ6XAM,5953
+sentry_sdk/serializer.py,sha256=W3HrcoJ9iMzHa6j9Nmu5Y7Qopbkon3ovxuPxRM5R4pU,13199
+sentry_sdk/session.py,sha256=ajYuxKt5cKfxl_OjcF0-g5IpowrJhvMe9rjTq1CaQac,5274
+sentry_sdk/sessions.py,sha256=-k0OGB28DoWbb7nIW4ML8R_fiF5nLnboWqObnfyZnWQ,8932
+sentry_sdk/spotlight.py,sha256=0bJrMpoT47xS2LJzpDfUZ5z5YEhdUla78r0YDtVm2Zo,12191
+sentry_sdk/traces.py,sha256=1oVGyMonl10vIrFokfD5UBHXQoPIu4rP2rtnKL3Ju6Y,4568
+sentry_sdk/tracing.py,sha256=OkIHt-BYfYhLe9MyEcTUlgN_n0CjqQn-BJ0K_9gnxaY,50512
+sentry_sdk/tracing_utils.py,sha256=aItaWdi76Q0-HGsILrkNesctXp76BkriTol-jXSZjfI,43300
+sentry_sdk/transport.py,sha256=P21HMCcD58PJfRNPA7Y39hlMqIFK5RsPCMwsy-QzXAQ,32141
+sentry_sdk/types.py,sha256=A92AqvfrGQZ9KY6FaUjKfL9F1HK7Ui3heQilVzfzYCs,1269
+sentry_sdk/utils.py,sha256=j6aRWZoMqiiN3Z_woIvw81E-rypx-CBxg5cQGmBbOIQ,65101
+sentry_sdk/worker.py,sha256=djRCygOJFRmdXBS4lRSF4RTljVZ47BWEal85WlTD0uo,4257

source/sentry_sdk-2.53.0.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.0)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

source/sentry_sdk-2.53.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[opentelemetry_propagator]
+sentry = sentry_sdk.integrations.opentelemetry:SentryPropagator

source/sentry_sdk-2.53.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Functional Software, Inc. dba Sentry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

source/sentry_sdk-2.53.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sentry_sdk

source/sentry_sdk/__init__.py

@@ -0,0 +1,67 @@
+from sentry_sdk import profiler
+from sentry_sdk import metrics
+from sentry_sdk.scope import Scope
+from sentry_sdk.transport import Transport, HttpTransport
+from sentry_sdk.client import Client
+
+from sentry_sdk.api import *  # noqa
+from sentry_sdk.consts import VERSION
+
+__all__ = [  # noqa
+    "Hub",
+    "Scope",
+    "Client",
+    "Transport",
+    "HttpTransport",
+    "VERSION",
+    "integrations",
+    # From sentry_sdk.api
+    "init",
+    "add_attachment",
+    "add_breadcrumb",
+    "capture_event",
+    "capture_exception",
+    "capture_message",
+    "configure_scope",
+    "continue_trace",
+    "flush",
+    "get_baggage",
+    "get_client",
+    "get_global_scope",
+    "get_isolation_scope",
+    "get_current_scope",
+    "get_current_span",
+    "get_traceparent",
+    "is_initialized",
+    "isolation_scope",
+    "last_event_id",
+    "new_scope",
+    "push_scope",
+    "set_context",
+    "set_extra",
+    "set_level",
+    "set_measurement",
+    "set_tag",
+    "set_tags",
+    "set_user",
+    "start_span",
+    "start_transaction",
+    "trace",
+    "monitor",
+    "logger",
+    "metrics",
+    "profiler",
+    "start_session",
+    "end_session",
+    "set_transaction_name",
+    "update_current_span",
+]
+
+# Initialize the debug support after everything is loaded
+from sentry_sdk.debug import init_debug_support
+
+init_debug_support()
+del init_debug_support
+
+# circular imports
+from sentry_sdk.hub import Hub

source/sentry_sdk/_batcher.py

@@ -0,0 +1,139 @@
+import os
+import random
+import threading
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, TypeVar, Generic
+
+from sentry_sdk.utils import format_timestamp, safe_repr, serialize_attribute
+from sentry_sdk.envelope import Envelope, Item, PayloadRef
+
+if TYPE_CHECKING:
+    from typing import Optional, Callable, Any
+
+T = TypeVar("T")
+
+
+class Batcher(Generic[T]):
+    MAX_BEFORE_FLUSH = 100
+    MAX_BEFORE_DROP = 1_000
+    FLUSH_WAIT_TIME = 5.0
+
+    TYPE = ""
+    CONTENT_TYPE = ""
+
+    def __init__(
+        self,
+        capture_func: "Callable[[Envelope], None]",
+        record_lost_func: "Callable[..., None]",
+    ) -> None:
+        self._buffer: "list[T]" = []
+        self._capture_func = capture_func
+        self._record_lost_func = record_lost_func
+        self._running = True
+        self._lock = threading.Lock()
+
+        self._flush_event: "threading.Event" = threading.Event()
+
+        self._flusher: "Optional[threading.Thread]" = None
+        self._flusher_pid: "Optional[int]" = None
+
+    def _ensure_thread(self) -> bool:
+        """For forking processes we might need to restart this thread.
+        This ensures that our process actually has that thread running.
+        """
+        if not self._running:
+            return False
+
+        pid = os.getpid()
+        if self._flusher_pid == pid:
+            return True
+
+        with self._lock:
+            # Recheck to make sure another thread didn't get here and start the
+            # the flusher in the meantime
+            if self._flusher_pid == pid:
+                return True
+
+            self._flusher_pid = pid
+
+            self._flusher = threading.Thread(target=self._flush_loop)
+            self._flusher.daemon = True
+
+            try:
+                self._flusher.start()
+            except RuntimeError:
+                # Unfortunately at this point the interpreter is in a state that no
+                # longer allows us to spawn a thread and we have to bail.
+                self._running = False
+                return False
+
+        return True
+
+    def _flush_loop(self) -> None:
+        while self._running:
+            self._flush_event.wait(self.FLUSH_WAIT_TIME + random.random())
+            self._flush_event.clear()
+            self._flush()
+
+    def add(self, item: "T") -> None:
+        if not self._ensure_thread() or self._flusher is None:
+            return None
+
+        with self._lock:
+            if len(self._buffer) >= self.MAX_BEFORE_DROP:
+                self._record_lost(item)
+                return None
+
+            self._buffer.append(item)
+            if len(self._buffer) >= self.MAX_BEFORE_FLUSH:
+                self._flush_event.set()
+
+    def kill(self) -> None:
+        if self._flusher is None:
+            return
+
+        self._running = False
+        self._flush_event.set()
+        self._flusher = None
+
+    def flush(self) -> None:
+        self._flush()
+
+    def _add_to_envelope(self, envelope: "Envelope") -> None:
+        envelope.add_item(
+            Item(
+                type=self.TYPE,
+                content_type=self.CONTENT_TYPE,
+                headers={
+                    "item_count": len(self._buffer),
+                },
+                payload=PayloadRef(
+                    json={
+                        "items": [
+                            self._to_transport_format(item) for item in self._buffer
+                        ]
+                    }
+                ),
+            )
+        )
+
+    def _flush(self) -> "Optional[Envelope]":
+        envelope = Envelope(
+            headers={"sent_at": format_timestamp(datetime.now(timezone.utc))}
+        )
+        with self._lock:
+            if len(self._buffer) == 0:
+                return None
+
+            self._add_to_envelope(envelope)
+            self._buffer.clear()
+
+        self._capture_func(envelope)
+        return envelope
+
+    def _record_lost(self, item: "T") -> None:
+        pass
+
+    @staticmethod
+    def _to_transport_format(item: "T") -> "Any":
+        pass

source/sentry_sdk/_compat.py

@@ -0,0 +1,94 @@
+import sys
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
+    from typing import TypeVar
+
+    T = TypeVar("T")
+
+
+PY37 = sys.version_info[0] == 3 and sys.version_info[1] >= 7
+PY38 = sys.version_info[0] == 3 and sys.version_info[1] >= 8
+PY310 = sys.version_info[0] == 3 and sys.version_info[1] >= 10
+PY311 = sys.version_info[0] == 3 and sys.version_info[1] >= 11
+
+
+def with_metaclass(meta: "Any", *bases: "Any") -> "Any":
+    class MetaClass(type):
+        def __new__(metacls: "Any", name: "Any", this_bases: "Any", d: "Any") -> "Any":
+            return meta(name, bases, d)
+
+    return type.__new__(MetaClass, "temporary_class", (), {})
+
+
+def check_uwsgi_thread_support() -> bool:
+    # We check two things here:
+    #
+    # 1. uWSGI doesn't run in threaded mode by default -- issue a warning if
+    #    that's the case.
+    #
+    # 2. Additionally, if uWSGI is running in preforking mode (default), it needs
+    #    the --py-call-uwsgi-fork-hooks option for the SDK to work properly. This
+    #    is because any background threads spawned before the main process is
+    #    forked are NOT CLEANED UP IN THE CHILDREN BY DEFAULT even if
+    #    --enable-threads is on. One has to explicitly provide
+    #    --py-call-uwsgi-fork-hooks to force uWSGI to run regular cpython
+    #    after-fork hooks that take care of cleaning up stale thread data.
+    try:
+        from uwsgi import opt  # type: ignore
+    except ImportError:
+        return True
+
+    from sentry_sdk.consts import FALSE_VALUES
+
+    def enabled(option: str) -> bool:
+        value = opt.get(option, False)
+        if isinstance(value, bool):
+            return value
+
+        if isinstance(value, bytes):
+            try:
+                value = value.decode()
+            except Exception:
+                pass
+
+        return value and str(value).lower() not in FALSE_VALUES
+
+    # When `threads` is passed in as a uwsgi option,
+    # `enable-threads` is implied on.
+    threads_enabled = "threads" in opt or enabled("enable-threads")
+    fork_hooks_on = enabled("py-call-uwsgi-fork-hooks")
+    lazy_mode = enabled("lazy-apps") or enabled("lazy")
+
+    if lazy_mode and not threads_enabled:
+        from warnings import warn
+
+        warn(
+            Warning(
+                "IMPORTANT: "
+                "We detected the use of uWSGI without thread support. "
+                "This might lead to unexpected issues. "
+                'Please run uWSGI with "--enable-threads" for full support.'
+            )
+        )
+
+        return False
+
+    elif not lazy_mode and (not threads_enabled or not fork_hooks_on):
+        from warnings import warn
+
+        warn(
+            Warning(
+                "IMPORTANT: "
+                "We detected the use of uWSGI in preforking mode without "
+                "thread support. This might lead to crashing workers. "
+                'Please run uWSGI with both "--enable-threads" and '
+                '"--py-call-uwsgi-fork-hooks" for full support.'
+            )
+        )
+
+        return False
+
+    return True

source/sentry_sdk/_init_implementation.py

@@ -0,0 +1,79 @@
+import warnings
+
+from typing import TYPE_CHECKING
+
+import sentry_sdk
+
+if TYPE_CHECKING:
+    from typing import Any, ContextManager, Optional
+
+    import sentry_sdk.consts
+
+
+class _InitGuard:
+    _CONTEXT_MANAGER_DEPRECATION_WARNING_MESSAGE = (
+        "Using the return value of sentry_sdk.init as a context manager "
+        "and manually calling the __enter__ and __exit__ methods on the "
+        "return value are deprecated. We are no longer maintaining this "
+        "functionality, and we will remove it in the next major release."
+    )
+
+    def __init__(self, client: "sentry_sdk.Client") -> None:
+        self._client = client
+
+    def __enter__(self) -> "_InitGuard":
+        warnings.warn(
+            self._CONTEXT_MANAGER_DEPRECATION_WARNING_MESSAGE,
+            stacklevel=2,
+            category=DeprecationWarning,
+        )
+
+        return self
+
+    def __exit__(self, exc_type: "Any", exc_value: "Any", tb: "Any") -> None:
+        warnings.warn(
+            self._CONTEXT_MANAGER_DEPRECATION_WARNING_MESSAGE,
+            stacklevel=2,
+            category=DeprecationWarning,
+        )
+
+        c = self._client
+        if c is not None:
+            c.close()
+
+
+def _check_python_deprecations() -> None:
+    # Since we're likely to deprecate Python versions in the future, I'm keeping
+    # this handy function around. Use this to detect the Python version used and
+    # to output logger.warning()s if it's deprecated.
+    pass
+
+
+def _init(*args: "Optional[str]", **kwargs: "Any") -> "ContextManager[Any]":
+    """Initializes the SDK and optionally integrations.
+
+    This takes the same arguments as the client constructor.
+    """
+    client = sentry_sdk.Client(*args, **kwargs)
+    sentry_sdk.get_global_scope().set_client(client)
+    _check_python_deprecations()
+    rv = _InitGuard(client)
+    return rv
+
+
+if TYPE_CHECKING:
+    # Make mypy, PyCharm and other static analyzers think `init` is a type to
+    # have nicer autocompletion for params.
+    #
+    # Use `ClientConstructor` to define the argument types of `init` and
+    # `ContextManager[Any]` to tell static analyzers about the return type.
+
+    class init(sentry_sdk.consts.ClientConstructor, _InitGuard):  # noqa: N801
+        pass
+
+else:
+    # Alias `init` for actual usage. Go through the lambda indirection to throw
+    # PyCharm off of the weakly typed signature (it would otherwise discover
+    # both the weakly typed signature of `_init` and our faked `init` type).
+
+    init = (lambda: _init)()

source/sentry_sdk/_log_batcher.py

@@ -0,0 +1,56 @@
+from typing import TYPE_CHECKING
+
+from sentry_sdk._batcher import Batcher
+from sentry_sdk.utils import serialize_attribute
+from sentry_sdk.envelope import Envelope, Item, PayloadRef
+
+if TYPE_CHECKING:
+    from typing import Any
+    from sentry_sdk._types import Log
+
+
+class LogBatcher(Batcher["Log"]):
+    MAX_BEFORE_FLUSH = 100
+    MAX_BEFORE_DROP = 1_000
+    FLUSH_WAIT_TIME = 5.0
+
+    TYPE = "log"
+    CONTENT_TYPE = "application/vnd.sentry.items.log+json"
+
+    @staticmethod
+    def _to_transport_format(item: "Log") -> "Any":
+        if "sentry.severity_number" not in item["attributes"]:
+            item["attributes"]["sentry.severity_number"] = item["severity_number"]
+        if "sentry.severity_text" not in item["attributes"]:
+            item["attributes"]["sentry.severity_text"] = item["severity_text"]
+
+        res = {
+            "timestamp": int(item["time_unix_nano"]) / 1.0e9,
+            "trace_id": item.get("trace_id", "00000000-0000-0000-0000-000000000000"),
+            "span_id": item.get("span_id"),
+            "level": str(item["severity_text"]),
+            "body": str(item["body"]),
+            "attributes": {
+                k: serialize_attribute(v) for (k, v) in item["attributes"].items()
+            },
+        }
+
+        return res
+
+    def _record_lost(self, item: "Log") -> None:
+        # Construct log envelope item without sending it to report lost bytes
+        log_item = Item(
+            type=self.TYPE,
+            content_type=self.CONTENT_TYPE,
+            headers={
+                "item_count": 1,
+            },
+            payload=PayloadRef(json={"items": [self._to_transport_format(item)]}),
+        )
+
+        self._record_lost_func(
+            reason="queue_overflow",
+            data_category="log_item",
+            item=log_item,
+            quantity=1,
+        )